Skip to content

L
linux
Commit cc628c96 authored by David S. Miller's avatar David S. Miller
Browse files
Options

Merge tag 'mlx5e-failsafe' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux 

Saeed Mahameed says:

====================
mlx5e-failsafe 27-03-2017

This series provides a fail-safe mechanism to allow safely re-configuring
mlx5e netdevice and provides a resiliency against sporadic
configuration failures.

To enable this we do some refactoring and code reorganizing to allow
breaking the drivers open/close flows to stages:
      open -> activate -> deactivate -> close.

In addition we need to allow creating fresh HW ring resources
(mlx5e_channels) with their own "new" set of parameters, while keeping
the current ones running and active until the new channels are
successfully created with the new configuration, and only then we can
safly replace (switch) old channels with new ones.

For that we introduce mlx5e_channels object and an API to manage it:
 - channels = open_channels(new_params):
   open fresh TX/RX channels
 - activate_channels(channels):
   redirect traffic to them and attach them to the netdev
 - deactivate_channes(channels)
   stop traffic and detach from netdev
 - close(channels)
   Free the TX/RX HW resources of those channels

With the above strategy it is straightforward to achieve the desired
behavior of fail-safe configuration.  In pseudo code:

make_new_config(new_params)
{
	old_channels = current_active_channels;
	new_channels = create_channels(new_params);
	if (!new_channels)
		return "Failed, but current channels are still active :)"

	deactivate_channels(old_channels); /* Can't fail */
	set_hw_new_state();                /* If needed  */
	activate_channels(new_channels);   /* Can't fail */
	close_channels(old_channels);
	current_active_channels = new_channels;

        return "SUCCESS";
}

At the top of this series, we change the following flows to be fail-safe:
ethtool:
   - ring parameters
   - coalesce parameters
   - tx copy break parameters
   - cqe compressing/moderation mode setting (priv flags)
ndos:
   - tc setup
   - set features: LRO
   - change mtu
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 95ed0edd 2e20a151
Hide whitespace changes
Inline Side-by-side
Showing with 984 additions and 789 deletions
drivers/net/ethernet/mellanox/mlx5/core/en.h
View file @ cc628c96
......@@ -182,15 +182,15 @@ enum mlx5e_priv_flag {
MLX5E_PFLAG_RX_CQE_COMPRESS = (1 << 1),
};
#define MLX5E_SET_PFLAG(priv, pflag, enable) \
#define MLX5E_SET_PFLAG(params, pflag, enable) \
do { \
if (enable) \
(priv)->params.pflags |= (pflag); \
(params)->pflags |= (pflag); \
else \
(priv)->params.pflags &= ~(pflag); \
(params)->pflags &= ~(pflag); \
} while (0)
#define MLX5E_GET_PFLAG(priv, pflag) (!!((priv)->params.pflags & (pflag)))
#define MLX5E_GET_PFLAG(params, pflag) (!!((params)->pflags & (pflag)))
#ifdef CONFIG_MLX5_CORE_EN_DCB
#define MLX5E_MAX_BW_ALLOC 100 /* Max percentage of BW allocation */
......@@ -213,7 +213,6 @@ struct mlx5e_params {
bool rx_cqe_compress_def;
struct mlx5e_cq_moder rx_cq_moderation;
struct mlx5e_cq_moder tx_cq_moderation;
u16 min_rx_wqes;
bool lro_en;
u32 lro_wqe_sz;
u16 tx_max_inline;
......@@ -225,6 +224,7 @@ struct mlx5e_params {
bool rx_am_enabled;
u32 lro_timeout;
u32 pflags;
struct bpf_prog *xdp_prog;
};
#ifdef CONFIG_MLX5_CORE_EN_DCB
......@@ -280,7 +280,6 @@ struct mlx5e_cq {
struct napi_struct *napi;
struct mlx5_core_cq mcq;
struct mlx5e_channel *channel;
struct mlx5e_priv *priv;
/* cqe decompression */
struct mlx5_cqe64 title;
......@@ -290,6 +289,7 @@ struct mlx5e_cq {
u16 decmprs_wqe_counter;
/* control */
struct mlx5_core_dev *mdev;
struct mlx5_frag_wq_ctrl wq_ctrl;
} ____cacheline_aligned_in_smp;
......@@ -357,7 +357,7 @@ struct mlx5e_txqsq {
/* control path */
struct mlx5_wq_ctrl wq_ctrl;
struct mlx5e_channel *channel;
int tc;
int txq_ix;
u32 rate_limit;
} ____cacheline_aligned_in_smp;
......@@ -533,7 +533,7 @@ struct mlx5e_rq {
u32 mpwqe_num_strides;
u32 rqn;
struct mlx5e_channel *channel;
struct mlx5e_priv *priv;
struct mlx5_core_dev *mdev;
struct mlx5_core_mkey umr_mkey;
} ____cacheline_aligned_in_smp;
......@@ -556,10 +556,18 @@ struct mlx5e_channel {
/* control */
struct mlx5e_priv *priv;
struct mlx5_core_dev *mdev;
struct mlx5e_tstamp *tstamp;
int ix;
int cpu;
};
struct mlx5e_channels {
struct mlx5e_channel **c;
unsigned int num;
struct mlx5e_params params;
};
enum mlx5e_traffic_types {
MLX5E_TT_IPV4_TCP,
MLX5E_TT_IPV6_TCP,
......@@ -709,34 +717,17 @@ enum {
MLX5E_NIC_PRIO
};
struct mlx5e_profile {
void (*init)(struct mlx5_core_dev *mdev,
struct net_device *netdev,
const struct mlx5e_profile *profile, void *ppriv);
void (*cleanup)(struct mlx5e_priv *priv);
int (*init_rx)(struct mlx5e_priv *priv);
void (*cleanup_rx)(struct mlx5e_priv *priv);
int (*init_tx)(struct mlx5e_priv *priv);
void (*cleanup_tx)(struct mlx5e_priv *priv);
void (*enable)(struct mlx5e_priv *priv);
void (*disable)(struct mlx5e_priv *priv);
void (*update_stats)(struct mlx5e_priv *priv);
int (*max_nch)(struct mlx5_core_dev *mdev);
int max_tc;
};
struct mlx5e_priv {
/* priv data path fields - start */
struct mlx5e_txqsq **txq_to_sq_map;
int channeltc_to_txq_map[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC];
struct bpf_prog *xdp_prog;
struct mlx5e_txqsq *txq2sq[MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC];
int channel_tc2txq[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC];
/* priv data path fields - end */
unsigned long state;
struct mutex state_lock; /* Protects Interface state */
struct mlx5e_rq drop_rq;
struct mlx5e_channel **channel;
struct mlx5e_channels channels;
u32 tisn[MLX5E_MAX_NUM_TC];
struct mlx5e_rqt indir_rqt;
struct mlx5e_tir indir_tir[MLX5E_NUM_INDIR_TIRS];
......@@ -746,7 +737,6 @@ struct mlx5e_priv {
struct mlx5e_flow_steering fs;
struct mlx5e_vxlan_db vxlan;
struct mlx5e_params params;
struct workqueue_struct *wq;
struct work_struct update_carrier_work;
struct work_struct set_rx_mode_work;
......@@ -766,6 +756,22 @@ struct mlx5e_priv {
void *ppriv;
};
struct mlx5e_profile {
void (*init)(struct mlx5_core_dev *mdev,
struct net_device *netdev,
const struct mlx5e_profile *profile, void *ppriv);
void (*cleanup)(struct mlx5e_priv *priv);
int (*init_rx)(struct mlx5e_priv *priv);
void (*cleanup_rx)(struct mlx5e_priv *priv);
int (*init_tx)(struct mlx5e_priv *priv);
void (*cleanup_tx)(struct mlx5e_priv *priv);
void (*enable)(struct mlx5e_priv *priv);
void (*disable)(struct mlx5e_priv *priv);
void (*update_stats)(struct mlx5e_priv *priv);
int (*max_nch)(struct mlx5_core_dev *mdev);
int max_tc;
};
void mlx5e_build_ptys2ethtool_map(void);
u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
......@@ -827,7 +833,7 @@ void mlx5e_pps_event_handler(struct mlx5e_priv *priv,
struct ptp_clock_event *event);
int mlx5e_hwstamp_set(struct net_device *dev, struct ifreq *ifr);
int mlx5e_hwstamp_get(struct net_device *dev, struct ifreq *ifr);
void mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool val);
int mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool val);
int mlx5e_vlan_rx_add_vid(struct net_device *dev, __always_unused __be16 proto,
u16 vid);
......@@ -836,14 +842,40 @@ int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __always_unused __be16 proto,
void mlx5e_enable_vlan_filter(struct mlx5e_priv *priv);
void mlx5e_disable_vlan_filter(struct mlx5e_priv *priv);
int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd);
int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd);
struct mlx5e_redirect_rqt_param {
bool is_rss;
union {
u32 rqn; /* Direct RQN (Non-RSS) */
struct {
u8 hfunc;
struct mlx5e_channels *channels;
} rss; /* RSS data */
};
};
int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix);
void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_priv *priv, void *tirc,
enum mlx5e_traffic_types tt);
int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
struct mlx5e_redirect_rqt_param rrp);
void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_params *params,
enum mlx5e_traffic_types tt,
void *tirc);
int mlx5e_open_locked(struct net_device *netdev);
int mlx5e_close_locked(struct net_device *netdev);
int mlx5e_open_channels(struct mlx5e_priv *priv,
struct mlx5e_channels *chs);
void mlx5e_close_channels(struct mlx5e_channels *chs);
/* Function pointer to be used to modify WH settings while
* switching channels
*/
typedef int (*mlx5e_fp_hw_modify)(struct mlx5e_priv *priv);
void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
struct mlx5e_channels *new_chs,
mlx5e_fp_hw_modify hw_modify);
void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
u32 *indirection_rqt, int len,
int num_channels);
......@@ -851,7 +883,8 @@ int mlx5e_get_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed);
void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params,
u8 cq_period_mode);
void mlx5e_set_rq_type_params(struct mlx5e_priv *priv, u8 rq_type);
void mlx5e_set_rq_type_params(struct mlx5_core_dev *mdev,
struct mlx5e_params *params, u8 rq_type);
static inline
struct mlx5e_tx_wqe *mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
......@@ -942,8 +975,7 @@ void mlx5e_destroy_tir(struct mlx5_core_dev *mdev,
struct mlx5e_tir *tir);
int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev);
void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev);
int mlx5e_refresh_tirs_self_loopback(struct mlx5_core_dev *mdev,
bool enable_uc_lb);
int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb);
struct mlx5_eswitch_rep;
int mlx5e_vport_rep_load(struct mlx5_eswitch *esw,
......
drivers/net/ethernet/mellanox/mlx5/core/en_clock.c
View file @ cc628c96
......@@ -90,6 +90,7 @@ int mlx5e_hwstamp_set(struct net_device *dev, struct ifreq *ifr)
{
struct mlx5e_priv *priv = netdev_priv(dev);
struct hwtstamp_config config;
int err;
if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
return -EOPNOTSUPP;
......@@ -111,7 +112,7 @@ int mlx5e_hwstamp_set(struct net_device *dev, struct ifreq *ifr)
switch (config.rx_filter) {
case HWTSTAMP_FILTER_NONE:
/* Reset CQE compression to Admin default */
mlx5e_modify_rx_cqe_compression_locked(priv, priv->params.rx_cqe_compress_def);
mlx5e_modify_rx_cqe_compression_locked(priv, priv->channels.params.rx_cqe_compress_def);
break;
case HWTSTAMP_FILTER_ALL:
case HWTSTAMP_FILTER_SOME:
......@@ -129,7 +130,12 @@ int mlx5e_hwstamp_set(struct net_device *dev, struct ifreq *ifr)
case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
/* Disable CQE compression */
netdev_warn(dev, "Disabling cqe compression");
mlx5e_modify_rx_cqe_compression_locked(priv, false);
err = mlx5e_modify_rx_cqe_compression_locked(priv, false);
if (err) {
netdev_err(dev, "Failed disabling cqe compression err=%d\n", err);
mutex_unlock(&priv->state_lock);
return err;
}
config.rx_filter = HWTSTAMP_FILTER_ALL;
break;
default:
......
drivers/net/ethernet/mellanox/mlx5/core/en_common.c
View file @ cc628c96
......@@ -136,18 +136,20 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev)
mlx5_core_dealloc_pd(mdev, res->pdn);
}
int mlx5e_refresh_tirs_self_loopback(struct mlx5_core_dev *mdev,
bool enable_uc_lb)
int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb)
{
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5e_tir *tir;
void *in;
int err = -ENOMEM;
u32 tirn = 0;
int inlen;
int err = 0;
void *in;
inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
in = mlx5_vzalloc(inlen);
if (!in)
return -ENOMEM;
goto out;
if (enable_uc_lb)
MLX5_SET(modify_tir_in, in, ctx.self_lb_block,
......@@ -156,13 +158,16 @@ int mlx5e_refresh_tirs_self_loopback(struct mlx5_core_dev *mdev,
MLX5_SET(modify_tir_in, in, bitmask.self_lb_en, 1);
list_for_each_entry(tir, &mdev->mlx5e_res.td.tirs_list, list) {
err = mlx5_core_modify_tir(mdev, tir->tirn, in, inlen);
tirn = tir->tirn;
err = mlx5_core_modify_tir(mdev, tirn, in, inlen);
if (err)
goto out;
}
out:
kvfree(in);
if (err)
netdev_err(priv->netdev, "refresh tir(0x%x) failed, %d\n", tirn, err);
return err;
}
drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
View file @ cc628c96
......@@ -152,12 +152,9 @@ static bool mlx5e_query_global_pause_combined(struct mlx5e_priv *priv)
}
#define MLX5E_NUM_Q_CNTRS(priv) (NUM_Q_COUNTERS * (!!priv->q_counter))
#define MLX5E_NUM_RQ_STATS(priv) \
(NUM_RQ_STATS * priv->params.num_channels * \
test_bit(MLX5E_STATE_OPENED, &priv->state))
#define MLX5E_NUM_RQ_STATS(priv) (NUM_RQ_STATS * (priv)->channels.num)
#define MLX5E_NUM_SQ_STATS(priv) \
(NUM_SQ_STATS * priv->params.num_channels * priv->params.num_tc * \
test_bit(MLX5E_STATE_OPENED, &priv->state))
(NUM_SQ_STATS * (priv)->channels.num * (priv)->channels.params.num_tc)
#define MLX5E_NUM_PFC_COUNTERS(priv) \
((mlx5e_query_global_pause_combined(priv) + hweight8(mlx5e_query_pfc_combined(priv))) * \
NUM_PPORT_PER_PRIO_PFC_COUNTERS)
......@@ -262,17 +259,17 @@ static void mlx5e_fill_stats_strings(struct mlx5e_priv *priv, uint8_t *data)
return;
/* per channel counters */
for (i = 0; i < priv->params.num_channels; i++)
for (i = 0; i < priv->channels.num; i++)
for (j = 0; j < NUM_RQ_STATS; j++)
sprintf(data + (idx++) * ETH_GSTRING_LEN,
rq_stats_desc[j].format, i);
for (tc = 0; tc < priv->params.num_tc; tc++)
for (i = 0; i < priv->params.num_channels; i++)
for (tc = 0; tc < priv->channels.params.num_tc; tc++)
for (i = 0; i < priv->channels.num; i++)
for (j = 0; j < NUM_SQ_STATS; j++)
sprintf(data + (idx++) * ETH_GSTRING_LEN,
sq_stats_desc[j].format,
priv->channeltc_to_txq_map[i][tc]);
priv->channel_tc2txq[i][tc]);
}
static void mlx5e_get_strings(struct net_device *dev,
......@@ -303,6 +300,7 @@ static void mlx5e_get_ethtool_stats(struct net_device *dev,
struct ethtool_stats *stats, u64 *data)
{
struct mlx5e_priv *priv = netdev_priv(dev);
struct mlx5e_channels *channels;
struct mlx5_priv *mlx5_priv;
int i, j, tc, prio, idx = 0;
unsigned long pfc_combined;
......@@ -313,6 +311,7 @@ static void mlx5e_get_ethtool_stats(struct net_device *dev,
mutex_lock(&priv->state_lock);
if (test_bit(MLX5E_STATE_OPENED, &priv->state))
mlx5e_update_stats(priv);
channels = &priv->channels;
mutex_unlock(&priv->state_lock);
for (i = 0; i < NUM_SW_COUNTERS; i++)
......@@ -382,16 +381,16 @@ static void mlx5e_get_ethtool_stats(struct net_device *dev,
return;
/* per channel counters */
for (i = 0; i < priv->params.num_channels; i++)
for (i = 0; i < channels->num; i++)
for (j = 0; j < NUM_RQ_STATS; j++)
data[idx++] =
MLX5E_READ_CTR64_CPU(&priv->channel[i]->rq.stats,
MLX5E_READ_CTR64_CPU(&channels->c[i]->rq.stats,
rq_stats_desc, j);
for (tc = 0; tc < priv->params.num_tc; tc++)
for (i = 0; i < priv->params.num_channels; i++)
for (tc = 0; tc < priv->channels.params.num_tc; tc++)
for (i = 0; i < channels->num; i++)
for (j = 0; j < NUM_SQ_STATS; j++)
data[idx++] = MLX5E_READ_CTR64_CPU(&priv->channel[i]->sq[tc].stats,
data[idx++] = MLX5E_READ_CTR64_CPU(&channels->c[i]->sq[tc].stats,
sq_stats_desc, j);
}
......@@ -406,8 +405,8 @@ static u32 mlx5e_rx_wqes_to_packets(struct mlx5e_priv *priv, int rq_wq_type,
if (rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
return num_wqe;
stride_size = 1 << priv->params.mpwqe_log_stride_sz;
num_strides = 1 << priv->params.mpwqe_log_num_strides;
stride_size = 1 << priv->channels.params.mpwqe_log_stride_sz;
num_strides = 1 << priv->channels.params.mpwqe_log_num_strides;
wqe_size = stride_size * num_strides;
packets_per_wqe = wqe_size /
......@@ -427,8 +426,8 @@ static u32 mlx5e_packets_to_rx_wqes(struct mlx5e_priv *priv, int rq_wq_type,
if (rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
return num_packets;
stride_size = 1 << priv->params.mpwqe_log_stride_sz;
num_strides = 1 << priv->params.mpwqe_log_num_strides;
stride_size = 1 << priv->channels.params.mpwqe_log_stride_sz;
num_strides = 1 << priv->channels.params.mpwqe_log_num_strides;
wqe_size = stride_size * num_strides;
num_packets = (1 << order_base_2(num_packets));
......@@ -443,26 +442,25 @@ static void mlx5e_get_ringparam(struct net_device *dev,
struct ethtool_ringparam *param)
{
struct mlx5e_priv *priv = netdev_priv(dev);
int rq_wq_type = priv->params.rq_wq_type;
int rq_wq_type = priv->channels.params.rq_wq_type;
param->rx_max_pending = mlx5e_rx_wqes_to_packets(priv, rq_wq_type,
1 << mlx5_max_log_rq_size(rq_wq_type));
param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE;
param->rx_pending = mlx5e_rx_wqes_to_packets(priv, rq_wq_type,
1 << priv->params.log_rq_size);
param->tx_pending = 1 << priv->params.log_sq_size;
1 << priv->channels.params.log_rq_size);
param->tx_pending = 1 << priv->channels.params.log_sq_size;
}
static int mlx5e_set_ringparam(struct net_device *dev,
struct ethtool_ringparam *param)
{
struct mlx5e_priv *priv = netdev_priv(dev);
bool was_opened;
int rq_wq_type = priv->params.rq_wq_type;
int rq_wq_type = priv->channels.params.rq_wq_type;
struct mlx5e_channels new_channels = {};
u32 rx_pending_wqes;
u32 min_rq_size;
u32 max_rq_size;
u16 min_rx_wqes;
u8 log_rq_size;
u8 log_sq_size;
u32 num_mtts;
......@@ -500,7 +498,7 @@ static int mlx5e_set_ringparam(struct net_device *dev,
}
num_mtts = MLX5E_REQUIRED_MTTS(rx_pending_wqes);
if (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
!MLX5E_VALID_NUM_MTTS(num_mtts)) {
netdev_info(dev, "%s: rx_pending (%d) request can't be satisfied, try to reduce.\n",
__func__, param->rx_pending);
......@@ -522,26 +520,29 @@ static int mlx5e_set_ringparam(struct net_device *dev,
log_rq_size = order_base_2(rx_pending_wqes);
log_sq_size = order_base_2(param->tx_pending);
min_rx_wqes = mlx5_min_rx_wqes(rq_wq_type, rx_pending_wqes);
if (log_rq_size == priv->params.log_rq_size &&
log_sq_size == priv->params.log_sq_size &&
min_rx_wqes == priv->params.min_rx_wqes)
if (log_rq_size == priv->channels.params.log_rq_size &&
log_sq_size == priv->channels.params.log_sq_size)
return 0;
mutex_lock(&priv->state_lock);
was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (was_opened)
mlx5e_close_locked(dev);
new_channels.params = priv->channels.params;
new_channels.params.log_rq_size = log_rq_size;
new_channels.params.log_sq_size = log_sq_size;
priv->params.log_rq_size = log_rq_size;
priv->params.log_sq_size = log_sq_size;
priv->params.min_rx_wqes = min_rx_wqes;
if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
priv->channels.params = new_channels.params;
goto unlock;
}
err = mlx5e_open_channels(priv, &new_channels);
if (err)
goto unlock;
if (was_opened)
err = mlx5e_open_locked(dev);
mlx5e_switch_priv_channels(priv, &new_channels, NULL);
unlock:
mutex_unlock(&priv->state_lock);
return err;
......@@ -553,7 +554,7 @@ static void mlx5e_get_channels(struct net_device *dev,
struct mlx5e_priv *priv = netdev_priv(dev);
ch->max_combined = priv->profile->max_nch(priv->mdev);
ch->combined_count = priv->params.num_channels;
ch->combined_count = priv->channels.params.num_channels;
}
static int mlx5e_set_channels(struct net_device *dev,
......@@ -561,8 +562,8 @@ static int mlx5e_set_channels(struct net_device *dev,
{
struct mlx5e_priv *priv = netdev_priv(dev);
unsigned int count = ch->combined_count;
struct mlx5e_channels new_channels = {};
bool arfs_enabled;
bool was_opened;
int err = 0;
if (!count) {
......@@ -571,27 +572,32 @@ static int mlx5e_set_channels(struct net_device *dev,
return -EINVAL;
}
if (priv->params.num_channels == count)
if (priv->channels.params.num_channels == count)
return 0;
mutex_lock(&priv->state_lock);
was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (was_opened)
mlx5e_close_locked(dev);
new_channels.params = priv->channels.params;
new_channels.params.num_channels = count;
mlx5e_build_default_indir_rqt(priv->mdev, new_channels.params.indirection_rqt,
MLX5E_INDIR_RQT_SIZE, count);
if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
priv->channels.params = new_channels.params;
goto out;
}
/* Create fresh channels with new parameters */
err = mlx5e_open_channels(priv, &new_channels);
if (err)
goto out;
arfs_enabled = dev->features & NETIF_F_NTUPLE;
if (arfs_enabled)
mlx5e_arfs_disable(priv);
priv->params.num_channels = count;
mlx5e_build_default_indir_rqt(priv->mdev, priv->params.indirection_rqt,
MLX5E_INDIR_RQT_SIZE, count);
if (was_opened)
err = mlx5e_open_locked(dev);
if (err)
goto out;
/* Switch to new channels, set new parameters and close old ones */
mlx5e_switch_priv_channels(priv, &new_channels, NULL);
if (arfs_enabled) {
err = mlx5e_arfs_enable(priv);
......@@ -614,49 +620,24 @@ static int mlx5e_get_coalesce(struct net_device *netdev,
if (!MLX5_CAP_GEN(priv->mdev, cq_moderation))
return -EOPNOTSUPP;
coal->rx_coalesce_usecs = priv->params.rx_cq_moderation.usec;
coal->rx_max_coalesced_frames = priv->params.rx_cq_moderation.pkts;
coal->tx_coalesce_usecs = priv->params.tx_cq_moderation.usec;
coal->tx_max_coalesced_frames = priv->params.tx_cq_moderation.pkts;
coal->use_adaptive_rx_coalesce = priv->params.rx_am_enabled;
coal->rx_coalesce_usecs = priv->channels.params.rx_cq_moderation.usec;
coal->rx_max_coalesced_frames = priv->channels.params.rx_cq_moderation.pkts;
coal->tx_coalesce_usecs = priv->channels.params.tx_cq_moderation.usec;
coal->tx_max_coalesced_frames = priv->channels.params.tx_cq_moderation.pkts;
coal->use_adaptive_rx_coalesce = priv->channels.params.rx_am_enabled;
return 0;
}
static int mlx5e_set_coalesce(struct net_device *netdev,
struct ethtool_coalesce *coal)
static void
mlx5e_set_priv_channels_coalesce(struct mlx5e_priv *priv, struct ethtool_coalesce *coal)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5e_channel *c;
bool restart =
!!coal->use_adaptive_rx_coalesce != priv->params.rx_am_enabled;
bool was_opened;
int err = 0;
int tc;
int i;
if (!MLX5_CAP_GEN(mdev, cq_moderation))
return -EOPNOTSUPP;
mutex_lock(&priv->state_lock);
was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (was_opened && restart) {
mlx5e_close_locked(netdev);
priv->params.rx_am_enabled = !!coal->use_adaptive_rx_coalesce;
}
priv->params.tx_cq_moderation.usec = coal->tx_coalesce_usecs;
priv->params.tx_cq_moderation.pkts = coal->tx_max_coalesced_frames;
priv->params.rx_cq_moderation.usec = coal->rx_coalesce_usecs;
priv->params.rx_cq_moderation.pkts = coal->rx_max_coalesced_frames;
if (!was_opened || restart)
goto out;
for (i = 0; i < priv->params.num_channels; ++i) {
c = priv->channel[i];
for (i = 0; i < priv->channels.num; ++i) {
struct mlx5e_channel *c = priv->channels.c[i];
for (tc = 0; tc < c->num_tc; tc++) {
mlx5_core_modify_cq_moderation(mdev,
......@@ -669,11 +650,50 @@ static int mlx5e_set_coalesce(struct net_device *netdev,
coal->rx_coalesce_usecs,
coal->rx_max_coalesced_frames);
}
}
out:
if (was_opened && restart)
err = mlx5e_open_locked(netdev);
static int mlx5e_set_coalesce(struct net_device *netdev,
struct ethtool_coalesce *coal)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5e_channels new_channels = {};
int err = 0;
bool reset;
if (!MLX5_CAP_GEN(mdev, cq_moderation))
return -EOPNOTSUPP;
mutex_lock(&priv->state_lock);
new_channels.params = priv->channels.params;
new_channels.params.tx_cq_moderation.usec = coal->tx_coalesce_usecs;
new_channels.params.tx_cq_moderation.pkts = coal->tx_max_coalesced_frames;
new_channels.params.rx_cq_moderation.usec = coal->rx_coalesce_usecs;
new_channels.params.rx_cq_moderation.pkts = coal->rx_max_coalesced_frames;
new_channels.params.rx_am_enabled = !!coal->use_adaptive_rx_coalesce;
if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
priv->channels.params = new_channels.params;
goto out;
}
/* we are opened */
reset = !!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_am_enabled;
if (!reset) {
mlx5e_set_priv_channels_coalesce(priv, coal);
priv->channels.params = new_channels.params;
goto out;
}
/* open fresh channels with new coal parameters */
err = mlx5e_open_channels(priv, &new_channels);
if (err)
goto out;
mlx5e_switch_priv_channels(priv, &new_channels, NULL);
out:
mutex_unlock(&priv->state_lock);
return err;
}
......@@ -968,7 +988,7 @@ static u32 mlx5e_get_rxfh_key_size(struct net_device *netdev)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
return sizeof(priv->params.toeplitz_hash_key);
return sizeof(priv->channels.params.toeplitz_hash_key);
}
static u32 mlx5e_get_rxfh_indir_size(struct net_device *netdev)
......@@ -982,15 +1002,15 @@ static int mlx5e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
struct mlx5e_priv *priv = netdev_priv(netdev);
if (indir)
memcpy(indir, priv->params.indirection_rqt,
sizeof(priv->params.indirection_rqt));
memcpy(indir, priv->channels.params.indirection_rqt,
sizeof(priv->channels.params.indirection_rqt));
if (key)
memcpy(key, priv->params.toeplitz_hash_key,
sizeof(priv->params.toeplitz_hash_key));
memcpy(key, priv->channels.params.toeplitz_hash_key,
sizeof(priv->channels.params.toeplitz_hash_key));
if (hfunc)
*hfunc = priv->params.rss_hfunc;
*hfunc = priv->channels.params.rss_hfunc;
return 0;
}
......@@ -1006,7 +1026,7 @@ static void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in, int inlen)
for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
memset(tirc, 0, ctxlen);
mlx5e_build_indir_tir_ctx_hash(priv, tirc, tt);
mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc);
mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in, inlen);
}
}
......@@ -1030,25 +1050,33 @@ static int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir,
mutex_lock(&priv->state_lock);
if (indir) {
u32 rqtn = priv->indir_rqt.rqtn;
memcpy(priv->params.indirection_rqt, indir,
sizeof(priv->params.indirection_rqt));
mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0);
}
if (hfunc != ETH_RSS_HASH_NO_CHANGE &&
hfunc != priv->params.rss_hfunc) {
priv->params.rss_hfunc = hfunc;
hfunc != priv->channels.params.rss_hfunc) {
priv->channels.params.rss_hfunc = hfunc;
hash_changed = true;
}
if (indir) {
memcpy(priv->channels.params.indirection_rqt, indir,
sizeof(priv->channels.params.indirection_rqt));
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
u32 rqtn = priv->indir_rqt.rqtn;
struct mlx5e_redirect_rqt_param rrp = {
.is_rss = true,
.rss.hfunc = priv->channels.params.rss_hfunc,
.rss.channels = &priv->channels
};
mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp);
}
}
if (key) {
memcpy(priv->params.toeplitz_hash_key, key,
sizeof(priv->params.toeplitz_hash_key));
memcpy(priv->channels.params.toeplitz_hash_key, key,
sizeof(priv->channels.params.toeplitz_hash_key));
hash_changed = hash_changed ||
priv->params.rss_hfunc == ETH_RSS_HASH_TOP;
priv->channels.params.rss_hfunc == ETH_RSS_HASH_TOP;
}
if (hash_changed)
......@@ -1069,7 +1097,7 @@ static int mlx5e_get_rxnfc(struct net_device *netdev,
switch (info->cmd) {
case ETHTOOL_GRXRINGS:
info->data = priv->params.num_channels;
info->data = priv->channels.params.num_channels;
break;
case ETHTOOL_GRXCLSRLCNT:
info->rule_cnt = priv->fs.ethtool.tot_num_rules;
......@@ -1097,7 +1125,7 @@ static int mlx5e_get_tunable(struct net_device *dev,
switch (tuna->id) {
case ETHTOOL_TX_COPYBREAK:
*(u32 *)data = priv->params.tx_max_inline;
*(u32 *)data = priv->channels.params.tx_max_inline;
break;
default:
err = -EINVAL;
......@@ -1113,9 +1141,11 @@ static int mlx5e_set_tunable(struct net_device *dev,
{
struct mlx5e_priv *priv = netdev_priv(dev);
struct mlx5_core_dev *mdev = priv->mdev;
bool was_opened;
u32 val;
struct mlx5e_channels new_channels = {};
int err = 0;
u32 val;
mutex_lock(&priv->state_lock);
switch (tuna->id) {
case ETHTOOL_TX_COPYBREAK:
......@@ -1125,24 +1155,26 @@ static int mlx5e_set_tunable(struct net_device *dev,
break;
}
mutex_lock(&priv->state_lock);
new_channels.params = priv->channels.params;
new_channels.params.tx_max_inline = val;
was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (was_opened)
mlx5e_close_locked(dev);
priv->params.tx_max_inline = val;
if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
priv->channels.params = new_channels.params;
break;
}
if (was_opened)
err = mlx5e_open_locked(dev);
err = mlx5e_open_channels(priv, &new_channels);
if (err)
break;
mlx5e_switch_priv_channels(priv, &new_channels, NULL);
mutex_unlock(&priv->state_lock);
break;
default:
err = -EINVAL;
break;
}
mutex_unlock(&priv->state_lock);
return err;
}
......@@ -1442,15 +1474,15 @@ static int set_pflag_rx_cqe_based_moder(struct net_device *netdev, bool enable)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5e_channels new_channels = {};
bool rx_mode_changed;
u8 rx_cq_period_mode;
int err = 0;
bool reset;
rx_cq_period_mode = enable ?
MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
rx_mode_changed = rx_cq_period_mode != priv->params.rx_cq_period_mode;
rx_mode_changed = rx_cq_period_mode != priv->channels.params.rx_cq_period_mode;
if (rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE &&
!MLX5_CAP_GEN(mdev, cq_period_start_from_cqe))
......@@ -1459,16 +1491,51 @@ static int set_pflag_rx_cqe_based_moder(struct net_device *netdev, bool enable)
if (!rx_mode_changed)
return 0;
reset = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (reset)
mlx5e_close_locked(netdev);
new_channels.params = priv->channels.params;
mlx5e_set_rx_cq_mode_params(&new_channels.params, rx_cq_period_mode);
mlx5e_set_rx_cq_mode_params(&priv->params, rx_cq_period_mode);
if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
priv->channels.params = new_channels.params;
return 0;
}
if (reset)
err = mlx5e_open_locked(netdev);
err = mlx5e_open_channels(priv, &new_channels);
if (err)
return err;
return err;
mlx5e_switch_priv_channels(priv, &new_channels, NULL);
return 0;
}
int mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool new_val)
{
bool curr_val = MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS);
struct mlx5e_channels new_channels = {};
int err = 0;
if (!MLX5_CAP_GEN(priv->mdev, cqe_compression))
return new_val ? -EOPNOTSUPP : 0;
if (curr_val == new_val)
return 0;
new_channels.params = priv->channels.params;
MLX5E_SET_PFLAG(&new_channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS, new_val);
mlx5e_set_rq_type_params(priv->mdev, &new_channels.params,
new_channels.params.rq_wq_type);
if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
priv->channels.params = new_channels.params;
return 0;
}
err = mlx5e_open_channels(priv, &new_channels);
if (err)
return err;
mlx5e_switch_priv_channels(priv, &new_channels, NULL);
return 0;
}
static int set_pflag_rx_cqe_compress(struct net_device *netdev,
......@@ -1486,8 +1553,7 @@ static int set_pflag_rx_cqe_compress(struct net_device *netdev,
}
mlx5e_modify_rx_cqe_compression_locked(priv, enable);
priv->params.rx_cqe_compress_def = enable;
mlx5e_set_rq_type_params(priv, priv->params.rq_wq_type);
priv->channels.params.rx_cqe_compress_def = enable;
return 0;
}
......@@ -1499,7 +1565,7 @@ static int mlx5e_handle_pflag(struct net_device *netdev,
{
struct mlx5e_priv *priv = netdev_priv(netdev);
bool enable = !!(wanted_flags & flag);
u32 changes = wanted_flags ^ priv->params.pflags;
u32 changes = wanted_flags ^ priv->channels.params.pflags;
int err;
if (!(changes & flag))
......@@ -1512,7 +1578,7 @@ static int mlx5e_handle_pflag(struct net_device *netdev,
return err;
}
MLX5E_SET_PFLAG(priv, flag, enable);
MLX5E_SET_PFLAG(&priv->channels.params, flag, enable);
return 0;
}
......@@ -1541,7 +1607,7 @@ static u32 mlx5e_get_priv_flags(struct net_device *netdev)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
return priv->params.pflags;
return priv->channels.params.pflags;
}
static int mlx5e_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
......
drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c
View file @ cc628c96
......@@ -390,7 +390,7 @@ static int validate_flow(struct mlx5e_priv *priv,
if (fs->location >= MAX_NUM_OF_ETHTOOL_RULES)
return -EINVAL;
if (fs->ring_cookie >= priv->params.num_channels &&
if (fs->ring_cookie >= priv->channels.params.num_channels &&
fs->ring_cookie != RX_CLS_FLOW_DISC)
return -EINVAL;
......
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
View file @ cc628c96
......@@ -44,14 +44,11 @@
struct mlx5e_rq_param {
u32 rqc[MLX5_ST_SZ_DW(rqc)];
struct mlx5_wq_param wq;
bool am_enabled;
};
struct mlx5e_sq_param {
u32 sqc[MLX5_ST_SZ_DW(sqc)];
struct mlx5_wq_param wq;
u16 max_inline;
u8 min_inline_mode;
};
struct mlx5e_cq_param {
......@@ -78,49 +75,47 @@ static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
MLX5_CAP_ETH(mdev, reg_umr_sq);
}
void mlx5e_set_rq_type_params(struct mlx5e_priv *priv, u8 rq_type)
void mlx5e_set_rq_type_params(struct mlx5_core_dev *mdev,
struct mlx5e_params *params, u8 rq_type)
{
priv->params.rq_wq_type = rq_type;
priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
switch (priv->params.rq_wq_type) {
params->rq_wq_type = rq_type;
params->lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
switch (params->rq_wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
priv->params.log_rq_size = is_kdump_kernel() ?
params->log_rq_size = is_kdump_kernel() ?
MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW :
MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
priv->params.mpwqe_log_stride_sz =
MLX5E_GET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS) ?
MLX5_MPWRQ_CQE_CMPRS_LOG_STRIDE_SZ(priv->mdev) :
MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(priv->mdev);
priv->params.mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
priv->params.mpwqe_log_stride_sz;
params->mpwqe_log_stride_sz =
MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) ?
MLX5_MPWRQ_CQE_CMPRS_LOG_STRIDE_SZ(mdev) :
MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
params->mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
params->mpwqe_log_stride_sz;
break;
default: /* MLX5_WQ_TYPE_LINKED_LIST */
priv->params.log_rq_size = is_kdump_kernel() ?
params->log_rq_size = is_kdump_kernel() ?
MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
/* Extra room needed for build_skb */
priv->params.lro_wqe_sz -= MLX5_RX_HEADROOM +
params->lro_wqe_sz -= MLX5_RX_HEADROOM +
SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
}
priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
BIT(priv->params.log_rq_size));
mlx5_core_info(priv->mdev,
"MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
BIT(priv->params.log_rq_size),
BIT(priv->params.mpwqe_log_stride_sz),
MLX5E_GET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS));
mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
BIT(params->log_rq_size),
BIT(params->mpwqe_log_stride_sz),
MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
}
static void mlx5e_set_rq_priv_params(struct mlx5e_priv *priv)
static void mlx5e_set_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
{
u8 rq_type = mlx5e_check_fragmented_striding_rq_cap(priv->mdev) &&
!priv->xdp_prog ?
u8 rq_type = mlx5e_check_fragmented_striding_rq_cap(mdev) &&
!params->xdp_prog ?
MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
MLX5_WQ_TYPE_LINKED_LIST;
mlx5e_set_rq_type_params(priv, rq_type);
mlx5e_set_rq_type_params(mdev, params, rq_type);
}
static void mlx5e_update_carrier(struct mlx5e_priv *priv)
......@@ -180,8 +175,10 @@ static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
int i, j;
memset(s, 0, sizeof(*s));
for (i = 0; i < priv->params.num_channels; i++) {
rq_stats = &priv->channel[i]->rq.stats;
for (i = 0; i < priv->channels.num; i++) {
struct mlx5e_channel *c = priv->channels.c[i];
rq_stats = &c->rq.stats;
s->rx_packets += rq_stats->packets;
s->rx_bytes += rq_stats->bytes;
......@@ -203,8 +200,8 @@ static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
s->rx_cache_empty += rq_stats->cache_empty;
s->rx_cache_busy += rq_stats->cache_busy;
for (j = 0; j < priv->params.num_tc; j++) {
sq_stats = &priv->channel[i]->sq[j].stats;
for (j = 0; j < priv->channels.params.num_tc; j++) {
sq_stats = &c->sq[j].stats;
s->tx_packets += sq_stats->packets;
s->tx_bytes += sq_stats->bytes;
......@@ -494,11 +491,10 @@ static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
kfree(rq->mpwqe.info);
}
static int mlx5e_create_umr_mkey(struct mlx5e_priv *priv,
static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
u64 npages, u8 page_shift,
struct mlx5_core_mkey *umr_mkey)
{
struct mlx5_core_dev *mdev = priv->mdev;
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
void *mkc;
u32 *in;
......@@ -532,21 +528,20 @@ static int mlx5e_create_umr_mkey(struct mlx5e_priv *priv,
return err;
}
static int mlx5e_create_rq_umr_mkey(struct mlx5e_rq *rq)
static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
{
struct mlx5e_priv *priv = rq->priv;
u64 num_mtts = MLX5E_REQUIRED_MTTS(BIT(priv->params.log_rq_size));
u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->wq));
return mlx5e_create_umr_mkey(priv, num_mtts, PAGE_SHIFT, &rq->umr_mkey);
return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey);
}
static int mlx5e_alloc_rq(struct mlx5e_channel *c,
struct mlx5e_rq_param *param,
struct mlx5e_params *params,
struct mlx5e_rq_param *rqp,
struct mlx5e_rq *rq)
{
struct mlx5e_priv *priv = c->priv;
struct mlx5_core_dev *mdev = priv->mdev;
void *rqc = param->rqc;
struct mlx5_core_dev *mdev = c->mdev;
void *rqc = rqp->rqc;
void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
u32 byte_count;
u32 frag_sz;
......@@ -555,9 +550,9 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
int err;
int i;
param->wq.db_numa_node = cpu_to_node(c->cpu);
rqp->wq.db_numa_node = cpu_to_node(c->cpu);
err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->wq,
&rq->wq_ctrl);
if (err)
return err;
......@@ -566,15 +561,15 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
wq_sz = mlx5_wq_ll_get_size(&rq->wq);
rq->wq_type = priv->params.rq_wq_type;
rq->wq_type = params->rq_wq_type;
rq->pdev = c->pdev;
rq->netdev = c->netdev;
rq->tstamp = &priv->tstamp;
rq->tstamp = c->tstamp;
rq->channel = c;
rq->ix = c->ix;
rq->priv = c->priv;
rq->mdev = mdev;
rq->xdp_prog = priv->xdp_prog ? bpf_prog_inc(priv->xdp_prog) : NULL;
rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
if (IS_ERR(rq->xdp_prog)) {
err = PTR_ERR(rq->xdp_prog);
rq->xdp_prog = NULL;
......@@ -589,9 +584,9 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
rq->rx_headroom = MLX5_RX_HEADROOM;
}
switch (priv->params.rq_wq_type) {
switch (rq->wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
if (mlx5e_is_vf_vport_rep(priv)) {
if (mlx5e_is_vf_vport_rep(c->priv)) {
err = -EINVAL;
goto err_rq_wq_destroy;
}
......@@ -600,13 +595,13 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
rq->mpwqe_stride_sz = BIT(priv->params.mpwqe_log_stride_sz);
rq->mpwqe_num_strides = BIT(priv->params.mpwqe_log_num_strides);
rq->mpwqe_stride_sz = BIT(params->mpwqe_log_stride_sz);
rq->mpwqe_num_strides = BIT(params->mpwqe_log_num_strides);
rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
byte_count = rq->buff.wqe_sz;
err = mlx5e_create_rq_umr_mkey(rq);
err = mlx5e_create_rq_umr_mkey(mdev, rq);
if (err)
goto err_rq_wq_destroy;
rq->mkey_be = cpu_to_be32(rq->umr_mkey.key);
......@@ -623,7 +618,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
goto err_rq_wq_destroy;
}
if (mlx5e_is_vf_vport_rep(priv))
if (mlx5e_is_vf_vport_rep(c->priv))
rq->handle_rx_cqe = mlx5e_handle_rx_cqe_rep;
else
rq->handle_rx_cqe = mlx5e_handle_rx_cqe;
......@@ -631,9 +626,9 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
rq->alloc_wqe = mlx5e_alloc_rx_wqe;
rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
rq->buff.wqe_sz = (priv->params.lro_en) ?
priv->params.lro_wqe_sz :
MLX5E_SW2HW_MTU(priv->netdev->mtu);
rq->buff.wqe_sz = params->lro_en ?
params->lro_wqe_sz :
MLX5E_SW2HW_MTU(c->netdev->mtu);
byte_count = rq->buff.wqe_sz;
/* calc the required page order */
......@@ -657,8 +652,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
}
INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
rq->am.mode = priv->params.rx_cq_period_mode;
rq->am.mode = params->rx_cq_period_mode;
rq->page_cache.head = 0;
rq->page_cache.tail = 0;
......@@ -685,7 +679,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
switch (rq->wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
mlx5e_rq_free_mpwqe_info(rq);
mlx5_core_destroy_mkey(rq->priv->mdev, &rq->umr_mkey);
mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey);
break;
default: /* MLX5_WQ_TYPE_LINKED_LIST */
kfree(rq->dma_info);
......@@ -700,10 +694,10 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
mlx5_wq_destroy(&rq->wq_ctrl);
}
static int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
static int mlx5e_create_rq(struct mlx5e_rq *rq,
struct mlx5e_rq_param *param)
{
struct mlx5e_priv *priv = rq->priv;
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5_core_dev *mdev = rq->mdev;
void *in;
void *rqc;
......@@ -724,7 +718,6 @@ static int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn);
MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
MLX5_SET(rqc, rqc, vsd, priv->params.vlan_strip_disable);
MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
MLX5_ADAPTER_PAGE_SHIFT);
MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
......@@ -743,8 +736,7 @@ static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
int next_state)
{
struct mlx5e_channel *c = rq->channel;
struct mlx5e_priv *priv = c->priv;
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5_core_dev *mdev = c->mdev;
void *in;
void *rqc;
......@@ -771,9 +763,7 @@ static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
{
struct mlx5e_channel *c = rq->channel;
struct mlx5e_priv *priv = c->priv;
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5_core_dev *mdev = c->mdev;
void *in;
void *rqc;
int inlen;
......@@ -801,23 +791,26 @@ static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
{
mlx5_core_destroy_rq(rq->priv->mdev, rq->rqn);
mlx5_core_destroy_rq(rq->mdev, rq->rqn);
}
static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
{
unsigned long exp_time = jiffies + msecs_to_jiffies(20000);
struct mlx5e_channel *c = rq->channel;
struct mlx5e_priv *priv = c->priv;
struct mlx5_wq_ll *wq = &rq->wq;
u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5_wq_ll_get_size(wq));
while (time_before(jiffies, exp_time)) {
if (wq->cur_sz >= priv->params.min_rx_wqes)
if (wq->cur_sz >= min_wqes)
return 0;
msleep(20);
}
netdev_warn(c->netdev, "Failed to get min RX wqes on RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
rq->rqn, wq->cur_sz, min_wqes);
return -ETIMEDOUT;
}
......@@ -843,15 +836,13 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
}
static int mlx5e_open_rq(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct mlx5e_rq_param *param,
struct mlx5e_rq *rq)
{
struct mlx5e_icosq *sq = &c->icosq;
u16 pi = sq->pc & sq->wq.sz_m1;
struct mlx5e_tx_wqe *nopwqe;
int err;
err = mlx5e_alloc_rq(c, param, rq);
err = mlx5e_alloc_rq(c, params, param, rq);
if (err)
return err;
......@@ -859,22 +850,16 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
if (err)
goto err_free_rq;
set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
if (err)
goto err_destroy_rq;
if (param->am_enabled)
if (params->rx_am_enabled)
set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
sq->db.ico_wqe[pi].num_wqebbs = 1;
nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
return 0;
err_destroy_rq:
clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
mlx5e_destroy_rq(rq);
err_free_rq:
mlx5e_free_rq(rq);
......@@ -882,12 +867,28 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
return err;
}
static void mlx5e_close_rq(struct mlx5e_rq *rq)
static void mlx5e_activate_rq(struct mlx5e_rq *rq)
{
struct mlx5e_icosq *sq = &rq->channel->icosq;
u16 pi = sq->pc & sq->wq.sz_m1;
struct mlx5e_tx_wqe *nopwqe;
set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
sq->db.ico_wqe[pi].num_wqebbs = 1;
nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
}
static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
{
clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
cancel_work_sync(&rq->am.work);
}
static void mlx5e_close_rq(struct mlx5e_rq *rq)
{
cancel_work_sync(&rq->am.work);
mlx5e_destroy_rq(rq);
mlx5e_free_rx_descs(rq);
mlx5e_free_rq(rq);
......@@ -913,19 +914,19 @@ static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
}
static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct mlx5e_sq_param *param,
struct mlx5e_xdpsq *sq)
{
void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
struct mlx5e_priv *priv = c->priv;
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5_core_dev *mdev = c->mdev;
int err;
sq->pdev = c->pdev;
sq->mkey_be = c->mkey_be;
sq->channel = c;
sq->uar_map = mdev->mlx5e_res.bfreg.map;
sq->min_inline_mode = param->min_inline_mode;
sq->min_inline_mode = params->tx_min_inline_mode;
param->wq.db_numa_node = cpu_to_node(c->cpu);
err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
......@@ -969,13 +970,11 @@ static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
}
static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
int tc,
struct mlx5e_sq_param *param,
struct mlx5e_icosq *sq)
{
void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
struct mlx5e_priv *priv = c->priv;
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5_core_dev *mdev = c->mdev;
int err;
sq->pdev = c->pdev;
......@@ -1038,24 +1037,23 @@ static int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
}
static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
int tc,
int txq_ix,
struct mlx5e_params *params,
struct mlx5e_sq_param *param,
struct mlx5e_txqsq *sq)
{
void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
struct mlx5e_priv *priv = c->priv;
struct mlx5_core_dev *mdev = priv->mdev;
int txq_ix;
struct mlx5_core_dev *mdev = c->mdev;
int err;
sq->pdev = c->pdev;
sq->tstamp = &priv->tstamp;
sq->tstamp = c->tstamp;
sq->mkey_be = c->mkey_be;
sq->channel = c;
sq->tc = tc;
sq->txq_ix = txq_ix;
sq->uar_map = mdev->mlx5e_res.bfreg.map;
sq->max_inline = param->max_inline;
sq->min_inline_mode = param->min_inline_mode;
sq->max_inline = params->tx_max_inline;
sq->min_inline_mode = params->tx_min_inline_mode;
param->wq.db_numa_node = cpu_to_node(c->cpu);
err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
......@@ -1067,10 +1065,6 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
if (err)
goto err_sq_wq_destroy;
txq_ix = c->ix + tc * priv->params.num_channels;
sq->txq = netdev_get_tx_queue(priv->netdev, txq_ix);
priv->txq_to_sq_map[txq_ix] = sq;
sq->edge = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;
return 0;
......@@ -1095,13 +1089,11 @@ struct mlx5e_create_sq_param {
u8 min_inline_mode;
};
static int mlx5e_create_sq(struct mlx5e_priv *priv,
static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
struct mlx5e_sq_param *param,
struct mlx5e_create_sq_param *csp,
u32 *sqn)
{
struct mlx5_core_dev *mdev = priv->mdev;
void *in;
void *sqc;
void *wq;
......@@ -1128,7 +1120,7 @@ static int mlx5e_create_sq(struct mlx5e_priv *priv,
MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
MLX5_SET(wq, wq, uar_page, priv->mdev->mlx5e_res.bfreg.index);
MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.bfreg.index);
MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift -
MLX5_ADAPTER_PAGE_SHIFT);
MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma);
......@@ -1149,12 +1141,9 @@ struct mlx5e_modify_sq_param {
int rl_index;
};
static int mlx5e_modify_sq(struct mlx5e_priv *priv,
u32 sqn,
static int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
struct mlx5e_modify_sq_param *p)
{
struct mlx5_core_dev *mdev = priv->mdev;
void *in;
void *sqc;
int inlen;
......@@ -1181,12 +1170,12 @@ static int mlx5e_modify_sq(struct mlx5e_priv *priv,
return err;
}
static void mlx5e_destroy_sq(struct mlx5e_priv *priv, u32 sqn)
static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
{
mlx5_core_destroy_sq(priv->mdev, sqn);
mlx5_core_destroy_sq(mdev, sqn);
}
static int mlx5e_create_sq_rdy(struct mlx5e_priv *priv,
static int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
struct mlx5e_sq_param *param,
struct mlx5e_create_sq_param *csp,
u32 *sqn)
......@@ -1194,44 +1183,50 @@ static int mlx5e_create_sq_rdy(struct mlx5e_priv *priv,
struct mlx5e_modify_sq_param msp = {0};
int err;
err = mlx5e_create_sq(priv, param, csp, sqn);
err = mlx5e_create_sq(mdev, param, csp, sqn);
if (err)
return err;
msp.curr_state = MLX5_SQC_STATE_RST;
msp.next_state = MLX5_SQC_STATE_RDY;
err = mlx5e_modify_sq(priv, *sqn, &msp);
err = mlx5e_modify_sq(mdev, *sqn, &msp);
if (err)
mlx5e_destroy_sq(priv, *sqn);
mlx5e_destroy_sq(mdev, *sqn);
return err;
}
static int mlx5e_set_sq_maxrate(struct net_device *dev,
struct mlx5e_txqsq *sq, u32 rate);
static int mlx5e_open_txqsq(struct mlx5e_channel *c,
int tc,
u32 tisn,
int txq_ix,
struct mlx5e_params *params,
struct mlx5e_sq_param *param,
struct mlx5e_txqsq *sq)
{
struct mlx5e_create_sq_param csp = {};
struct mlx5e_priv *priv = c->priv;
u32 tx_rate;
int err;
err = mlx5e_alloc_txqsq(c, tc, param, sq);
err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq);
if (err)
return err;
csp.tisn = priv->tisn[sq->tc];
csp.tisn = tisn;
csp.tis_lst_sz = 1;
csp.cqn = sq->cq.mcq.cqn;
csp.wq_ctrl = &sq->wq_ctrl;
csp.min_inline_mode = sq->min_inline_mode;
set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
err = mlx5e_create_sq_rdy(c->priv, param, &csp, &sq->sqn);
err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
if (err)
goto err_free_txqsq;
netdev_tx_reset_queue(sq->txq);
netif_tx_start_queue(sq->txq);
tx_rate = c->priv->tx_rates[sq->txq_ix];
if (tx_rate)
mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
return 0;
err_free_txqsq:
......@@ -1241,6 +1236,14 @@ static int mlx5e_open_txqsq(struct mlx5e_channel *c,
return err;
}
static void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
{
sq->txq = netdev_get_tx_queue(sq->channel->netdev, sq->txq_ix);
set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
netdev_tx_reset_queue(sq->txq);
netif_tx_start_queue(sq->txq);
}
static inline void netif_tx_disable_queue(struct netdev_queue *txq)
{
__netif_tx_lock_bh(txq);
......@@ -1248,11 +1251,9 @@ static inline void netif_tx_disable_queue(struct netdev_queue *txq)
__netif_tx_unlock_bh(txq);
}
static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
{
struct mlx5e_channel *c = sq->channel;
struct mlx5e_priv *priv = c->priv;
struct mlx5_core_dev *mdev = priv->mdev;
clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
/* prevent netif_tx_wake_queue */
......@@ -1268,8 +1269,14 @@ static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
nop = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nop->ctrl);
}
}
mlx5e_destroy_sq(priv, sq->sqn);
static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
{
struct mlx5e_channel *c = sq->channel;
struct mlx5_core_dev *mdev = c->mdev;
mlx5e_destroy_sq(mdev, sq->sqn);
if (sq->rate_limit)
mlx5_rl_remove_rate(mdev, sq->rate_limit);
mlx5e_free_txqsq_descs(sq);
......@@ -1277,22 +1284,22 @@ static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
}
static int mlx5e_open_icosq(struct mlx5e_channel *c,
int tc,
struct mlx5e_params *params,
struct mlx5e_sq_param *param,
struct mlx5e_icosq *sq)
{
struct mlx5e_create_sq_param csp = {};
int err;
err = mlx5e_alloc_icosq(c, tc, param, sq);
err = mlx5e_alloc_icosq(c, param, sq);
if (err)
return err;
csp.cqn = sq->cq.mcq.cqn;
csp.wq_ctrl = &sq->wq_ctrl;
csp.min_inline_mode = param->min_inline_mode;
csp.min_inline_mode = params->tx_min_inline_mode;
set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
err = mlx5e_create_sq_rdy(c->priv, param, &csp, &sq->sqn);
err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
if (err)
goto err_free_icosq;
......@@ -1312,32 +1319,32 @@ static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
napi_synchronize(&c->napi);
mlx5e_destroy_sq(c->priv, sq->sqn);
mlx5e_destroy_sq(c->mdev, sq->sqn);
mlx5e_free_icosq(sq);
}
static int mlx5e_open_xdpsq(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct mlx5e_sq_param *param,
struct mlx5e_xdpsq *sq)
{
unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
struct mlx5e_create_sq_param csp = {};
struct mlx5e_priv *priv = c->priv;
unsigned int inline_hdr_sz = 0;
int err;
int i;
err = mlx5e_alloc_xdpsq(c, param, sq);
err = mlx5e_alloc_xdpsq(c, params, param, sq);
if (err)
return err;
csp.tis_lst_sz = 1;
csp.tisn = priv->tisn[0]; /* tc = 0 */
csp.tisn = c->priv->tisn[0]; /* tc = 0 */
csp.cqn = sq->cq.mcq.cqn;
csp.wq_ctrl = &sq->wq_ctrl;
csp.min_inline_mode = sq->min_inline_mode;
set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
err = mlx5e_create_sq_rdy(c->priv, param, &csp, &sq->sqn);
err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
if (err)
goto err_free_xdpsq;
......@@ -1376,7 +1383,7 @@ static void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
napi_synchronize(&c->napi);
mlx5e_destroy_sq(c->priv, sq->sqn);
mlx5e_destroy_sq(c->mdev, sq->sqn);
mlx5e_free_xdpsq_descs(sq);
mlx5e_free_xdpsq(sq);
}
......@@ -1385,8 +1392,7 @@ static int mlx5e_alloc_cq(struct mlx5e_channel *c,
struct mlx5e_cq_param *param,
struct mlx5e_cq *cq)
{
struct mlx5e_priv *priv = c->priv;
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5_core_dev *mdev = c->mdev;
struct mlx5_core_cq *mcq = &cq->mcq;
int eqn_not_used;
unsigned int irqn;
......@@ -1423,7 +1429,7 @@ static int mlx5e_alloc_cq(struct mlx5e_channel *c,
}
cq->channel = c;
cq->priv = priv;
cq->mdev = mdev;
return 0;
}
......@@ -1435,8 +1441,7 @@ static void mlx5e_free_cq(struct mlx5e_cq *cq)
static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
{
struct mlx5e_priv *priv = cq->priv;
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5_core_dev *mdev = cq->mdev;
struct mlx5_core_cq *mcq = &cq->mcq;
void *in;
......@@ -1482,20 +1487,16 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
{
struct mlx5e_priv *priv = cq->priv;
struct mlx5_core_dev *mdev = priv->mdev;
mlx5_core_destroy_cq(mdev, &cq->mcq);
mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
}
static int mlx5e_open_cq(struct mlx5e_channel *c,
struct mlx5e_cq_moder moder,
struct mlx5e_cq_param *param,
struct mlx5e_cq *cq,
struct mlx5e_cq_moder moderation)
struct mlx5e_cq *cq)
{
struct mlx5_core_dev *mdev = c->mdev;
int err;
struct mlx5e_priv *priv = c->priv;
struct mlx5_core_dev *mdev = priv->mdev;
err = mlx5e_alloc_cq(c, param, cq);
if (err)
......@@ -1506,9 +1507,7 @@ static int mlx5e_open_cq(struct mlx5e_channel *c,
goto err_free_cq;
if (MLX5_CAP_GEN(mdev, cq_moderation))
mlx5_core_modify_cq_moderation(mdev, &cq->mcq,
moderation.usec,
moderation.pkts);
mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts);
return 0;
err_free_cq:
......@@ -1529,15 +1528,15 @@ static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
}
static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct mlx5e_channel_param *cparam)
{
struct mlx5e_priv *priv = c->priv;
int err;
int tc;
for (tc = 0; tc < c->num_tc; tc++) {
err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
priv->params.tx_cq_moderation);
err = mlx5e_open_cq(c, params->tx_cq_moderation,
&cparam->tx_cq, &c->sq[tc].cq);
if (err)
goto err_close_tx_cqs;
}
......@@ -1560,13 +1559,17 @@ static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
}
static int mlx5e_open_sqs(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct mlx5e_channel_param *cparam)
{
int err;
int tc;
for (tc = 0; tc < c->num_tc; tc++) {
err = mlx5e_open_txqsq(c, tc, &cparam->sq, &c->sq[tc]);
for (tc = 0; tc < params->num_tc; tc++) {
int txq_ix = c->ix + tc * params->num_channels;
err = mlx5e_open_txqsq(c, c->priv->tisn[tc], txq_ix,
params, &cparam->sq, &c->sq[tc]);
if (err)
goto err_close_sqs;
}
......@@ -1588,15 +1591,6 @@ static void mlx5e_close_sqs(struct mlx5e_channel *c)
mlx5e_close_txqsq(&c->sq[tc]);
}
static void mlx5e_build_channeltc_to_txq_map(struct mlx5e_priv *priv, int ix)
{
int i;
for (i = 0; i < priv->profile->max_tc; i++)
priv->channeltc_to_txq_map[ix][i] =
ix + i * priv->params.num_channels;
}
static int mlx5e_set_sq_maxrate(struct net_device *dev,
struct mlx5e_txqsq *sq, u32 rate)
{
......@@ -1629,7 +1623,7 @@ static int mlx5e_set_sq_maxrate(struct net_device *dev,
msp.next_state = MLX5_SQC_STATE_RDY;
msp.rl_index = rl_index;
msp.rl_update = true;
err = mlx5e_modify_sq(priv, sq->sqn, &msp);
err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
if (err) {
netdev_err(dev, "Failed configuring rate %u: %d\n",
rate, err);
......@@ -1647,7 +1641,7 @@ static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
{
struct mlx5e_priv *priv = netdev_priv(dev);
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5e_txqsq *sq = priv->txq_to_sq_map[index];
struct mlx5e_txqsq *sq = priv->txq2sq[index];
int err = 0;
if (!mlx5_rl_is_supported(mdev)) {
......@@ -1683,88 +1677,69 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
}
static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
struct mlx5e_params *params,
struct mlx5e_channel_param *cparam,
struct mlx5e_channel **cp)
{
struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
struct mlx5e_cq_moder icocq_moder = {0, 0};
struct net_device *netdev = priv->netdev;
struct mlx5e_cq_moder rx_cq_profile;
int cpu = mlx5e_get_cpu(priv, ix);
struct mlx5e_channel *c;
int err;
int i;
c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
if (!c)
return -ENOMEM;
c->priv = priv;
c->mdev = priv->mdev;
c->tstamp = &priv->tstamp;
c->ix = ix;
c->cpu = cpu;
c->pdev = &priv->mdev->pdev->dev;
c->netdev = priv->netdev;
c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
c->num_tc = priv->params.num_tc;
c->xdp = !!priv->xdp_prog;
if (priv->params.rx_am_enabled)
rx_cq_profile = mlx5e_am_get_def_profile(priv->params.rx_cq_period_mode);
else
rx_cq_profile = priv->params.rx_cq_moderation;
mlx5e_build_channeltc_to_txq_map(priv, ix);
c->num_tc = params->num_tc;
c->xdp = !!params->xdp_prog;
netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
err = mlx5e_open_cq(c, &cparam->icosq_cq, &c->icosq.cq, icosq_cq_moder);
err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
if (err)
goto err_napi_del;
err = mlx5e_open_tx_cqs(c, cparam);
err = mlx5e_open_tx_cqs(c, params, cparam);
if (err)
goto err_close_icosq_cq;
err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
rx_cq_profile);
err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq);
if (err)
goto err_close_tx_cqs;
/* XDP SQ CQ params are same as normal TXQ sq CQ params */
err = c->xdp ? mlx5e_open_cq(c, &cparam->tx_cq, &c->rq.xdpsq.cq,
priv->params.tx_cq_moderation) : 0;
err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,
&cparam->tx_cq, &c->rq.xdpsq.cq) : 0;
if (err)
goto err_close_rx_cq;
napi_enable(&c->napi);
err = mlx5e_open_icosq(c, 0, &cparam->icosq, &c->icosq);
err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq);
if (err)
goto err_disable_napi;
err = mlx5e_open_sqs(c, cparam);
err = mlx5e_open_sqs(c, params, cparam);
if (err)
goto err_close_icosq;
for (i = 0; i < priv->params.num_tc; i++) {
u32 txq_ix = priv->channeltc_to_txq_map[ix][i];
if (priv->tx_rates[txq_ix]) {
struct mlx5e_txqsq *sq = priv->txq_to_sq_map[txq_ix];
mlx5e_set_sq_maxrate(priv->netdev, sq,
priv->tx_rates[txq_ix]);
}
}
err = c->xdp ? mlx5e_open_xdpsq(c, &cparam->xdp_sq, &c->rq.xdpsq) : 0;
err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq) : 0;
if (err)
goto err_close_sqs;
err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
err = mlx5e_open_rq(c, params, &cparam->rq, &c->rq);
if (err)
goto err_close_xdp_sq;
netif_set_xps_queue(netdev, get_cpu_mask(c->cpu), ix);
*cp = c;
return 0;
......@@ -1799,6 +1774,25 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
return err;
}
static void mlx5e_activate_channel(struct mlx5e_channel *c)
{
int tc;
for (tc = 0; tc < c->num_tc; tc++)
mlx5e_activate_txqsq(&c->sq[tc]);
mlx5e_activate_rq(&c->rq);
netif_set_xps_queue(c->netdev, get_cpu_mask(c->cpu), c->ix);
}
static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
{
int tc;
mlx5e_deactivate_rq(&c->rq);
for (tc = 0; tc < c->num_tc; tc++)
mlx5e_deactivate_txqsq(&c->sq[tc]);
}
static void mlx5e_close_channel(struct mlx5e_channel *c)
{
mlx5e_close_rq(&c->rq);
......@@ -1818,17 +1812,16 @@ static void mlx5e_close_channel(struct mlx5e_channel *c)
}
static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
struct mlx5e_params *params,
struct mlx5e_rq_param *param)
{
void *rqc = param->rqc;
void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
switch (priv->params.rq_wq_type) {
switch (params->rq_wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
MLX5_SET(wq, wq, log_wqe_num_of_strides,
priv->params.mpwqe_log_num_strides - 9);
MLX5_SET(wq, wq, log_wqe_stride_size,
priv->params.mpwqe_log_stride_sz - 6);
MLX5_SET(wq, wq, log_wqe_num_of_strides, params->mpwqe_log_num_strides - 9);
MLX5_SET(wq, wq, log_wqe_stride_size, params->mpwqe_log_stride_sz - 6);
MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
break;
default: /* MLX5_WQ_TYPE_LINKED_LIST */
......@@ -1837,14 +1830,13 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe)));
MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
MLX5_SET(wq, wq, log_wq_sz, params->log_rq_size);
MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn);
MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable);
param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
param->wq.linear = 1;
param->am_enabled = priv->params.rx_am_enabled;
}
static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
......@@ -1869,16 +1861,14 @@ static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
}
static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
struct mlx5e_params *params,
struct mlx5e_sq_param *param)
{
void *sqc = param->sqc;
void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
mlx5e_build_sq_param_common(priv, param);
MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
param->max_inline = priv->params.tx_max_inline;
param->min_inline_mode = priv->params.tx_min_inline_mode;
MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
}
static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
......@@ -1890,37 +1880,40 @@ static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
}
static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
struct mlx5e_params *params,
struct mlx5e_cq_param *param)
{
void *cqc = param->cqc;
u8 log_cq_size;
switch (priv->params.rq_wq_type) {
switch (params->rq_wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
log_cq_size = priv->params.log_rq_size +
priv->params.mpwqe_log_num_strides;
log_cq_size = params->log_rq_size + params->mpwqe_log_num_strides;
break;
default: /* MLX5_WQ_TYPE_LINKED_LIST */
log_cq_size = priv->params.log_rq_size;
log_cq_size = params->log_rq_size;
}
MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
if (MLX5E_GET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
MLX5_SET(cqc, cqc, cqe_comp_en, 1);
}
mlx5e_build_common_cq_param(priv, param);
param->cq_period_mode = priv->params.rx_cq_period_mode;
if (params->rx_am_enabled)
params->rx_cq_moderation =
mlx5e_am_get_def_profile(params->rx_cq_period_mode);
}
static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
struct mlx5e_params *params,
struct mlx5e_cq_param *param)
{
void *cqc = param->cqc;
MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size);
mlx5e_build_common_cq_param(priv, param);
......@@ -1928,8 +1921,8 @@ static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
}
static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
struct mlx5e_cq_param *param,
u8 log_wq_size)
u8 log_wq_size,
struct mlx5e_cq_param *param)
{
void *cqc = param->cqc;
......@@ -1941,8 +1934,8 @@ static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
}
static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
struct mlx5e_sq_param *param,
u8 log_wq_size)
u8 log_wq_size,
struct mlx5e_sq_param *param)
{
void *sqc = param->sqc;
void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
......@@ -1954,156 +1947,116 @@ static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
}
static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
struct mlx5e_params *params,
struct mlx5e_sq_param *param)
{
void *sqc = param->sqc;
void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
mlx5e_build_sq_param_common(priv, param);
MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
param->max_inline = priv->params.tx_max_inline;
param->min_inline_mode = priv->params.tx_min_inline_mode;
MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
}
static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam)
static void mlx5e_build_channel_param(struct mlx5e_priv *priv,
struct mlx5e_params *params,
struct mlx5e_channel_param *cparam)
{
u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
mlx5e_build_rq_param(priv, &cparam->rq);
mlx5e_build_sq_param(priv, &cparam->sq);
mlx5e_build_xdpsq_param(priv, &cparam->xdp_sq);
mlx5e_build_icosq_param(priv, &cparam->icosq, icosq_log_wq_sz);
mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
mlx5e_build_ico_cq_param(priv, &cparam->icosq_cq, icosq_log_wq_sz);
mlx5e_build_rq_param(priv, params, &cparam->rq);
mlx5e_build_sq_param(priv, params, &cparam->sq);
mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);
mlx5e_build_rx_cq_param(priv, params, &cparam->rx_cq);
mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq);
mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq);
}
static int mlx5e_open_channels(struct mlx5e_priv *priv)
int mlx5e_open_channels(struct mlx5e_priv *priv,
struct mlx5e_channels *chs)
{
struct mlx5e_channel_param *cparam;
int nch = priv->params.num_channels;
int err = -ENOMEM;
int i;
int j;
priv->channel = kcalloc(nch, sizeof(struct mlx5e_channel *),
GFP_KERNEL);
priv->txq_to_sq_map = kcalloc(nch * priv->params.num_tc,
sizeof(struct mlx5e_sq *), GFP_KERNEL);
chs->num = chs->params.num_channels;
chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
cparam = kzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
if (!chs->c || !cparam)
goto err_free;
if (!priv->channel || !priv->txq_to_sq_map || !cparam)
goto err_free_txq_to_sq_map;
mlx5e_build_channel_param(priv, cparam);
for (i = 0; i < nch; i++) {
err = mlx5e_open_channel(priv, i, cparam, &priv->channel[i]);
if (err)
goto err_close_channels;
}
for (j = 0; j < nch; j++) {
err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
mlx5e_build_channel_param(priv, &chs->params, cparam);
for (i = 0; i < chs->num; i++) {
err = mlx5e_open_channel(priv, i, &chs->params, cparam, &chs->c[i]);
if (err)
goto err_close_channels;
}
/* FIXME: This is a W/A for tx timeout watch dog false alarm when
* polling for inactive tx queues.
*/
netif_tx_start_all_queues(priv->netdev);
kfree(cparam);
return 0;
err_close_channels:
for (i--; i >= 0; i--)
mlx5e_close_channel(priv->channel[i]);
mlx5e_close_channel(chs->c[i]);
err_free_txq_to_sq_map:
kfree(priv->txq_to_sq_map);
kfree(priv->channel);
err_free:
kfree(chs->c);
kfree(cparam);
chs->num = 0;
return err;
}
static void mlx5e_close_channels(struct mlx5e_priv *priv)
static void mlx5e_activate_channels(struct mlx5e_channels *chs)
{
int i;
/* FIXME: This is a W/A only for tx timeout watch dog false alarm when
* polling for inactive tx queues.
*/
netif_tx_stop_all_queues(priv->netdev);
netif_tx_disable(priv->netdev);
for (i = 0; i < priv->params.num_channels; i++)
mlx5e_close_channel(priv->channel[i]);
kfree(priv->txq_to_sq_map);
kfree(priv->channel);
}
static int mlx5e_rx_hash_fn(int hfunc)
{
return (hfunc == ETH_RSS_HASH_TOP) ?
MLX5_RX_HASH_FN_TOEPLITZ :
MLX5_RX_HASH_FN_INVERTED_XOR8;
for (i = 0; i < chs->num; i++)
mlx5e_activate_channel(chs->c[i]);
}
static int mlx5e_bits_invert(unsigned long a, int size)
static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
{
int inv = 0;
int err = 0;
int i;
for (i = 0; i < size; i++)
inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
for (i = 0; i < chs->num; i++) {
err = mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq);
if (err)
break;
}
return inv;
return err;
}
static void mlx5e_fill_indir_rqt_rqns(struct mlx5e_priv *priv, void *rqtc)
static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
{
int i;
for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) {
int ix = i;
u32 rqn;
if (priv->params.rss_hfunc == ETH_RSS_HASH_XOR)
ix = mlx5e_bits_invert(i, MLX5E_LOG_INDIR_RQT_SIZE);
ix = priv->params.indirection_rqt[ix];
rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
priv->channel[ix]->rq.rqn :
priv->drop_rq.rqn;
MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
}
for (i = 0; i < chs->num; i++)
mlx5e_deactivate_channel(chs->c[i]);
}
static void mlx5e_fill_direct_rqt_rqn(struct mlx5e_priv *priv, void *rqtc,
int ix)
void mlx5e_close_channels(struct mlx5e_channels *chs)
{
u32 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
priv->channel[ix]->rq.rqn :
priv->drop_rq.rqn;
int i;
MLX5_SET(rqtc, rqtc, rq_num[0], rqn);
for (i = 0; i < chs->num; i++)
mlx5e_close_channel(chs->c[i]);
kfree(chs->c);
chs->num = 0;
}
static int mlx5e_create_rqt(struct mlx5e_priv *priv, int sz,
int ix, struct mlx5e_rqt *rqt)
static int
mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, struct mlx5e_rqt *rqt)
{
struct mlx5_core_dev *mdev = priv->mdev;
void *rqtc;
int inlen;
int err;
u32 *in;
int i;
inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
in = mlx5_vzalloc(inlen);
......@@ -2115,10 +2068,8 @@ static int mlx5e_create_rqt(struct mlx5e_priv *priv, int sz,
MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
if (sz > 1) /* RSS */
mlx5e_fill_indir_rqt_rqns(priv, rqtc);
else
mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
for (i = 0; i < sz; i++)
MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
if (!err)
......@@ -2138,7 +2089,7 @@ static int mlx5e_create_indirect_rqts(struct mlx5e_priv *priv)
{
struct mlx5e_rqt *rqt = &priv->indir_rqt;
return mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, 0, rqt);
return mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, rqt);
}
int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
......@@ -2149,7 +2100,7 @@ int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
rqt = &priv->direct_tir[ix].rqt;
err = mlx5e_create_rqt(priv, 1 /*size */, ix, rqt);
err = mlx5e_create_rqt(priv, 1 /*size */, rqt);
if (err)
goto err_destroy_rqts;
}
......@@ -2163,7 +2114,49 @@ int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
return err;
}
int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix)
static int mlx5e_rx_hash_fn(int hfunc)
{
return (hfunc == ETH_RSS_HASH_TOP) ?
MLX5_RX_HASH_FN_TOEPLITZ :
MLX5_RX_HASH_FN_INVERTED_XOR8;
}
static int mlx5e_bits_invert(unsigned long a, int size)
{
int inv = 0;
int i;
for (i = 0; i < size; i++)
inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
return inv;
}
static void mlx5e_fill_rqt_rqns(struct mlx5e_priv *priv, int sz,
struct mlx5e_redirect_rqt_param rrp, void *rqtc)
{
int i;
for (i = 0; i < sz; i++) {
u32 rqn;
if (rrp.is_rss) {
int ix = i;
if (rrp.rss.hfunc == ETH_RSS_HASH_XOR)
ix = mlx5e_bits_invert(i, ilog2(sz));
ix = priv->channels.params.indirection_rqt[ix];
rqn = rrp.rss.channels->c[ix]->rq.rqn;
} else {
rqn = rrp.rqn;
}
MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
}
}
int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
struct mlx5e_redirect_rqt_param rrp)
{
struct mlx5_core_dev *mdev = priv->mdev;
void *rqtc;
......@@ -2179,41 +2172,78 @@ int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix)
rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
if (sz > 1) /* RSS */
mlx5e_fill_indir_rqt_rqns(priv, rqtc);
else
mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
mlx5e_fill_rqt_rqns(priv, sz, rrp, rqtc);
err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
kvfree(in);
return err;
}
static void mlx5e_redirect_rqts(struct mlx5e_priv *priv)
static u32 mlx5e_get_direct_rqn(struct mlx5e_priv *priv, int ix,
struct mlx5e_redirect_rqt_param rrp)
{
if (!rrp.is_rss)
return rrp.rqn;
if (ix >= rrp.rss.channels->num)
return priv->drop_rq.rqn;
return rrp.rss.channels->c[ix]->rq.rqn;
}
static void mlx5e_redirect_rqts(struct mlx5e_priv *priv,
struct mlx5e_redirect_rqt_param rrp)
{
u32 rqtn;
int ix;
if (priv->indir_rqt.enabled) {
/* RSS RQ table */
rqtn = priv->indir_rqt.rqtn;
mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0);
mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp);
}
for (ix = 0; ix < priv->params.num_channels; ix++) {
for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
struct mlx5e_redirect_rqt_param direct_rrp = {
.is_rss = false,
.rqn = mlx5e_get_direct_rqn(priv, ix, rrp)
};
/* Direct RQ Tables */
if (!priv->direct_tir[ix].rqt.enabled)
continue;
rqtn = priv->direct_tir[ix].rqt.rqtn;
mlx5e_redirect_rqt(priv, rqtn, 1, ix);
mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp);
}
}
static void mlx5e_build_tir_ctx_lro(void *tirc, struct mlx5e_priv *priv)
static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv *priv,
struct mlx5e_channels *chs)
{
struct mlx5e_redirect_rqt_param rrp = {
.is_rss = true,
.rss.channels = chs,
.rss.hfunc = chs->params.rss_hfunc
};
mlx5e_redirect_rqts(priv, rrp);
}
static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv *priv)
{
struct mlx5e_redirect_rqt_param drop_rrp = {
.is_rss = false,
.rqn = priv->drop_rq.rqn
};
mlx5e_redirect_rqts(priv, drop_rrp);
}
static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc)
{
if (!priv->params.lro_en)
if (!params->lro_en)
return;
#define ROUGH_MAX_L2_L3_HDR_SZ 256
......@@ -2222,13 +2252,13 @@ static void mlx5e_build_tir_ctx_lro(void *tirc, struct mlx5e_priv *priv)
MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
(priv->params.lro_wqe_sz -
ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
MLX5_SET(tirc, tirc, lro_timeout_period_usecs, priv->params.lro_timeout);
(params->lro_wqe_sz - ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout);
}
void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_priv *priv, void *tirc,
enum mlx5e_traffic_types tt)
void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_params *params,
enum mlx5e_traffic_types tt,
void *tirc)
{
void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
......@@ -2244,16 +2274,15 @@ void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_priv *priv, void *tirc,
MLX5_HASH_FIELD_SEL_DST_IP |\
MLX5_HASH_FIELD_SEL_IPSEC_SPI)
MLX5_SET(tirc, tirc, rx_hash_fn,
mlx5e_rx_hash_fn(priv->params.rss_hfunc));
if (priv->params.rss_hfunc == ETH_RSS_HASH_TOP) {
MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_rx_hash_fn(params->rss_hfunc));
if (params->rss_hfunc == ETH_RSS_HASH_TOP) {
void *rss_key = MLX5_ADDR_OF(tirc, tirc,
rx_hash_toeplitz_key);
size_t len = MLX5_FLD_SZ_BYTES(tirc,
rx_hash_toeplitz_key);
MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
memcpy(rss_key, priv->params.toeplitz_hash_key, len);
memcpy(rss_key, params->toeplitz_hash_key, len);
}
switch (tt) {
......@@ -2358,7 +2387,7 @@ static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
mlx5e_build_tir_ctx_lro(tirc, priv);
mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in,
......@@ -2408,9 +2437,9 @@ static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu)
*mtu = MLX5E_HW2SW_MTU(hw_mtu);
}
static int mlx5e_set_dev_port_mtu(struct net_device *netdev)
static int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
struct net_device *netdev = priv->netdev;
u16 mtu;
int err;
......@@ -2430,8 +2459,8 @@ static int mlx5e_set_dev_port_mtu(struct net_device *netdev)
static void mlx5e_netdev_set_tcs(struct net_device *netdev)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
int nch = priv->params.num_channels;
int ntc = priv->params.num_tc;
int nch = priv->channels.params.num_channels;
int ntc = priv->channels.params.num_tc;
int tc;
netdev_reset_tc(netdev);
......@@ -2448,53 +2477,112 @@ static void mlx5e_netdev_set_tcs(struct net_device *netdev)
netdev_set_tc_queue(netdev, tc, nch, 0);
}
static void mlx5e_build_channels_tx_maps(struct mlx5e_priv *priv)
{
struct mlx5e_channel *c;
struct mlx5e_txqsq *sq;
int i, tc;
for (i = 0; i < priv->channels.num; i++)
for (tc = 0; tc < priv->profile->max_tc; tc++)
priv->channel_tc2txq[i][tc] = i + tc * priv->channels.num;
for (i = 0; i < priv->channels.num; i++) {
c = priv->channels.c[i];
for (tc = 0; tc < c->num_tc; tc++) {
sq = &c->sq[tc];
priv->txq2sq[sq->txq_ix] = sq;
}
}
}
static void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
{
int num_txqs = priv->channels.num * priv->channels.params.num_tc;
struct net_device *netdev = priv->netdev;
mlx5e_netdev_set_tcs(netdev);
if (netdev->real_num_tx_queues != num_txqs)
netif_set_real_num_tx_queues(netdev, num_txqs);
if (netdev->real_num_rx_queues != priv->channels.num)
netif_set_real_num_rx_queues(netdev, priv->channels.num);
mlx5e_build_channels_tx_maps(priv);
mlx5e_activate_channels(&priv->channels);
netif_tx_start_all_queues(priv->netdev);
if (MLX5_CAP_GEN(priv->mdev, vport_group_manager))
mlx5e_add_sqs_fwd_rules(priv);
mlx5e_wait_channels_min_rx_wqes(&priv->channels);
mlx5e_redirect_rqts_to_channels(priv, &priv->channels);
}
static void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
{
mlx5e_redirect_rqts_to_drop(priv);
if (MLX5_CAP_GEN(priv->mdev, vport_group_manager))
mlx5e_remove_sqs_fwd_rules(priv);
/* FIXME: This is a W/A only for tx timeout watch dog false alarm when
* polling for inactive tx queues.
*/
netif_tx_stop_all_queues(priv->netdev);
netif_tx_disable(priv->netdev);
mlx5e_deactivate_channels(&priv->channels);
}
void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
struct mlx5e_channels *new_chs,
mlx5e_fp_hw_modify hw_modify)
{
struct net_device *netdev = priv->netdev;
int new_num_txqs;
new_num_txqs = new_chs->num * new_chs->params.num_tc;
netif_carrier_off(netdev);
if (new_num_txqs < netdev->real_num_tx_queues)
netif_set_real_num_tx_queues(netdev, new_num_txqs);
mlx5e_deactivate_priv_channels(priv);
mlx5e_close_channels(&priv->channels);
priv->channels = *new_chs;
/* New channels are ready to roll, modify HW settings if needed */
if (hw_modify)
hw_modify(priv);
mlx5e_refresh_tirs(priv, false);
mlx5e_activate_priv_channels(priv);
mlx5e_update_carrier(priv);
}
int mlx5e_open_locked(struct net_device *netdev)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
struct mlx5_core_dev *mdev = priv->mdev;
int num_txqs;
int err;
set_bit(MLX5E_STATE_OPENED, &priv->state);
mlx5e_netdev_set_tcs(netdev);
num_txqs = priv->params.num_channels * priv->params.num_tc;
netif_set_real_num_tx_queues(netdev, num_txqs);
netif_set_real_num_rx_queues(netdev, priv->params.num_channels);
err = mlx5e_open_channels(priv);
if (err) {
netdev_err(netdev, "%s: mlx5e_open_channels failed, %d\n",
__func__, err);
err = mlx5e_open_channels(priv, &priv->channels);
if (err)
goto err_clear_state_opened_flag;
}
err = mlx5e_refresh_tirs_self_loopback(priv->mdev, false);
if (err) {
netdev_err(netdev, "%s: mlx5e_refresh_tirs_self_loopback_enable failed, %d\n",
__func__, err);
goto err_close_channels;
}
mlx5e_redirect_rqts(priv);
mlx5e_refresh_tirs(priv, false);
mlx5e_activate_priv_channels(priv);
mlx5e_update_carrier(priv);
mlx5e_timestamp_init(priv);
#ifdef CONFIG_RFS_ACCEL
priv->netdev->rx_cpu_rmap = priv->mdev->rmap;
#endif
if (priv->profile->update_stats)
queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
err = mlx5e_add_sqs_fwd_rules(priv);
if (err)
goto err_close_channels;
}
return 0;
err_close_channels:
mlx5e_close_channels(priv);
err_clear_state_opened_flag:
clear_bit(MLX5E_STATE_OPENED, &priv->state);
return err;
......@@ -2515,7 +2603,6 @@ int mlx5e_open(struct net_device *netdev)
int mlx5e_close_locked(struct net_device *netdev)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
struct mlx5_core_dev *mdev = priv->mdev;
/* May already be CLOSED in case a previous configuration operation
* (e.g RX/TX queue size change) that involves close&open failed.
......@@ -2525,13 +2612,10 @@ int mlx5e_close_locked(struct net_device *netdev)
clear_bit(MLX5E_STATE_OPENED, &priv->state);
if (MLX5_CAP_GEN(mdev, vport_group_manager))
mlx5e_remove_sqs_fwd_rules(priv);
mlx5e_timestamp_cleanup(priv);
netif_carrier_off(priv->netdev);
mlx5e_redirect_rqts(priv);
mlx5e_close_channels(priv);
mlx5e_deactivate_priv_channels(priv);
mlx5e_close_channels(&priv->channels);
return 0;
}
......@@ -2551,11 +2635,10 @@ int mlx5e_close(struct net_device *netdev)
return err;
}
static int mlx5e_alloc_drop_rq(struct mlx5e_priv *priv,
static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
struct mlx5e_rq *rq,
struct mlx5e_rq_param *param)
{
struct mlx5_core_dev *mdev = priv->mdev;
void *rqc = param->rqc;
void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
int err;
......@@ -2567,16 +2650,15 @@ static int mlx5e_alloc_drop_rq(struct mlx5e_priv *priv,
if (err)
return err;
rq->priv = priv;
rq->mdev = mdev;
return 0;
}
static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev,
struct mlx5e_cq *cq,
struct mlx5e_cq_param *param)
{
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5_core_cq *mcq = &cq->mcq;
int eqn_not_used;
unsigned int irqn;
......@@ -2599,24 +2681,22 @@ static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
mcq->event = mlx5e_cq_error_event;
mcq->irqn = irqn;
cq->priv = priv;
cq->mdev = mdev;
return 0;
}
static int mlx5e_open_drop_rq(struct mlx5e_priv *priv)
static int mlx5e_open_drop_rq(struct mlx5_core_dev *mdev,
struct mlx5e_rq *drop_rq)
{
struct mlx5e_cq_param cq_param;
struct mlx5e_rq_param rq_param;
struct mlx5e_rq *rq = &priv->drop_rq;
struct mlx5e_cq *cq = &priv->drop_rq.cq;
struct mlx5e_cq_param cq_param = {};
struct mlx5e_rq_param rq_param = {};
struct mlx5e_cq *cq = &drop_rq->cq;
int err;
memset(&cq_param, 0, sizeof(cq_param));
memset(&rq_param, 0, sizeof(rq_param));
mlx5e_build_drop_rq_param(&rq_param);
err = mlx5e_alloc_drop_cq(priv, cq, &cq_param);
err = mlx5e_alloc_drop_cq(mdev, cq, &cq_param);
if (err)
return err;
......@@ -2624,34 +2704,34 @@ static int mlx5e_open_drop_rq(struct mlx5e_priv *priv)
if (err)
goto err_free_cq;
err = mlx5e_alloc_drop_rq(priv, rq, &rq_param);
err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
if (err)
goto err_destroy_cq;
err = mlx5e_create_rq(rq, &rq_param);
err = mlx5e_create_rq(drop_rq, &rq_param);
if (err)
goto err_free_rq;
return 0;
err_free_rq:
mlx5e_free_rq(&priv->drop_rq);
mlx5e_free_rq(drop_rq);
err_destroy_cq:
mlx5e_destroy_cq(&priv->drop_rq.cq);
mlx5e_destroy_cq(cq);
err_free_cq:
mlx5e_free_cq(&priv->drop_rq.cq);
mlx5e_free_cq(cq);
return err;
}
static void mlx5e_close_drop_rq(struct mlx5e_priv *priv)
static void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
{
mlx5e_destroy_rq(&priv->drop_rq);
mlx5e_free_rq(&priv->drop_rq);
mlx5e_destroy_cq(&priv->drop_rq.cq);
mlx5e_free_cq(&priv->drop_rq.cq);
mlx5e_destroy_rq(drop_rq);
mlx5e_free_rq(drop_rq);
mlx5e_destroy_cq(&drop_rq->cq);
mlx5e_free_cq(&drop_rq->cq);
}
static int mlx5e_create_tis(struct mlx5e_priv *priv, int tc)
......@@ -2702,24 +2782,24 @@ void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
mlx5e_destroy_tis(priv, tc);
}
static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
enum mlx5e_traffic_types tt)
static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv,
enum mlx5e_traffic_types tt,
u32 *tirc)
{
MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
mlx5e_build_tir_ctx_lro(tirc, priv);
mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
mlx5e_build_indir_tir_ctx_hash(priv, tirc, tt);
mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc);
}
static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
u32 rqtn)
static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc)
{
MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
mlx5e_build_tir_ctx_lro(tirc, priv);
mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
MLX5_SET(tirc, tirc, indirect_table, rqtn);
......@@ -2744,7 +2824,7 @@ static int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
memset(in, 0, inlen);
tir = &priv->indir_tir[tt];
tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
mlx5e_build_indir_tir_ctx(priv, tirc, tt);
mlx5e_build_indir_tir_ctx(priv, tt, tirc);
err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
if (err)
goto err_destroy_tirs;
......@@ -2782,8 +2862,7 @@ int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)
memset(in, 0, inlen);
tir = &priv->direct_tir[ix];
tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
mlx5e_build_direct_tir_ctx(priv, tirc,
priv->direct_tir[ix].rqt.rqtn);
mlx5e_build_direct_tir_ctx(priv, priv->direct_tir[ix].rqt.rqtn, tirc);
err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
if (err)
goto err_destroy_ch_tirs;
......@@ -2819,16 +2898,13 @@ void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);
}
int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd)
int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
{
int err = 0;
int i;
if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
return 0;
for (i = 0; i < priv->params.num_channels; i++) {
err = mlx5e_modify_rq_vsd(&priv->channel[i]->rq, vsd);
for (i = 0; i < chs->num; i++) {
err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
if (err)
return err;
}
......@@ -2839,7 +2915,7 @@ int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd)
static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
bool was_opened;
struct mlx5e_channels new_channels = {};
int err = 0;
if (tc && tc != MLX5E_MAX_NUM_TC)
......@@ -2847,17 +2923,21 @@ static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
mutex_lock(&priv->state_lock);
was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (was_opened)
mlx5e_close_locked(priv->netdev);
new_channels.params = priv->channels.params;
new_channels.params.num_tc = tc ? tc : 1;
priv->params.num_tc = tc ? tc : 1;
if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
priv->channels.params = new_channels.params;
goto out;
}
if (was_opened)
err = mlx5e_open_locked(priv->netdev);
err = mlx5e_open_channels(priv, &new_channels);
if (err)
goto out;
mlx5e_switch_priv_channels(priv, &new_channels, NULL);
out:
mutex_unlock(&priv->state_lock);
return err;
}
......@@ -2974,26 +3054,31 @@ typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
static int set_feature_lro(struct net_device *netdev, bool enable)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
bool was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
int err;
struct mlx5e_channels new_channels = {};
int err = 0;
bool reset;
mutex_lock(&priv->state_lock);
if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
mlx5e_close_locked(priv->netdev);
reset = (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST);
reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state);
priv->params.lro_en = enable;
err = mlx5e_modify_tirs_lro(priv);
if (err) {
netdev_err(netdev, "lro modify failed, %d\n", err);
priv->params.lro_en = !enable;
new_channels.params = priv->channels.params;
new_channels.params.lro_en = enable;
if (!reset) {
priv->channels.params = new_channels.params;
err = mlx5e_modify_tirs_lro(priv);
goto out;
}
if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
mlx5e_open_locked(priv->netdev);
err = mlx5e_open_channels(priv, &new_channels);
if (err)
goto out;
mlx5e_switch_priv_channels(priv, &new_channels, mlx5e_modify_tirs_lro);
out:
mutex_unlock(&priv->state_lock);
return err;
}
......@@ -3033,15 +3118,19 @@ static int set_feature_rx_all(struct net_device *netdev, bool enable)
static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
int err;
int err = 0;
mutex_lock(&priv->state_lock);
priv->params.vlan_strip_disable = !enable;
err = mlx5e_modify_rqs_vsd(priv, !enable);
priv->channels.params.vlan_strip_disable = !enable;
if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
goto unlock;
err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
if (err)
priv->params.vlan_strip_disable = enable;
priv->channels.params.vlan_strip_disable = enable;
unlock:
mutex_unlock(&priv->state_lock);
return err;
......@@ -3112,28 +3201,38 @@ static int mlx5e_set_features(struct net_device *netdev,
static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
bool was_opened;
struct mlx5e_channels new_channels = {};
int curr_mtu;
int err = 0;
bool reset;
mutex_lock(&priv->state_lock);
reset = !priv->params.lro_en &&
(priv->params.rq_wq_type !=
reset = !priv->channels.params.lro_en &&
(priv->channels.params.rq_wq_type !=
MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (was_opened && reset)
mlx5e_close_locked(netdev);
reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state);
curr_mtu = netdev->mtu;
netdev->mtu = new_mtu;
mlx5e_set_dev_port_mtu(netdev);
if (was_opened && reset)
err = mlx5e_open_locked(netdev);
if (!reset) {
mlx5e_set_dev_port_mtu(priv);
goto out;
}
mutex_unlock(&priv->state_lock);
new_channels.params = priv->channels.params;
err = mlx5e_open_channels(priv, &new_channels);
if (err) {
netdev->mtu = curr_mtu;
goto out;
}
mlx5e_switch_priv_channels(priv, &new_channels, mlx5e_set_dev_port_mtu);
out:
mutex_unlock(&priv->state_lock);
return err;
}
......@@ -3338,8 +3437,8 @@ static void mlx5e_tx_timeout(struct net_device *dev)
netdev_err(dev, "TX timeout detected\n");
for (i = 0; i < priv->params.num_channels * priv->params.num_tc; i++) {
struct mlx5e_txqsq *sq = priv->txq_to_sq_map[i];
for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) {
struct mlx5e_txqsq *sq = priv->txq2sq[i];
if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i)))
continue;
......@@ -3371,7 +3470,7 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
/* no need for full reset when exchanging programs */
reset = (!priv->xdp_prog || !prog);
reset = (!priv->channels.params.xdp_prog || !prog);
if (was_opened && reset)
mlx5e_close_locked(netdev);
......@@ -3379,7 +3478,7 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
/* num_channels is invariant here, so we can take the
* batched reference right upfront.
*/
prog = bpf_prog_add(prog, priv->params.num_channels);
prog = bpf_prog_add(prog, priv->channels.num);
if (IS_ERR(prog)) {
err = PTR_ERR(prog);
goto unlock;
......@@ -3389,12 +3488,12 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
/* exchange programs, extra prog reference we got from caller
* as long as we don't fail from this point onwards.
*/
old_prog = xchg(&priv->xdp_prog, prog);
old_prog = xchg(&priv->channels.params.xdp_prog, prog);
if (old_prog)
bpf_prog_put(old_prog);
if (reset) /* change RQ type according to priv->xdp_prog */
mlx5e_set_rq_priv_params(priv);
mlx5e_set_rq_params(priv->mdev, &priv->channels.params);
if (was_opened && reset)
mlx5e_open_locked(netdev);
......@@ -3405,8 +3504,8 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
/* exchanging programs w/o reset, we update ref counts on behalf
* of the channels RQs here.
*/
for (i = 0; i < priv->params.num_channels; i++) {
struct mlx5e_channel *c = priv->channel[i];
for (i = 0; i < priv->channels.num; i++) {
struct mlx5e_channel *c = priv->channels.c[i];
clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
napi_synchronize(&c->napi);
......@@ -3432,7 +3531,7 @@ static bool mlx5e_xdp_attached(struct net_device *dev)
{
struct mlx5e_priv *priv = netdev_priv(dev);
return !!priv->xdp_prog;
return !!priv->channels.params.xdp_prog;
}
static int mlx5e_xdp(struct net_device *dev, struct netdev_xdp *xdp)
......@@ -3455,10 +3554,12 @@ static int mlx5e_xdp(struct net_device *dev, struct netdev_xdp *xdp)
static void mlx5e_netpoll(struct net_device *dev)
{
struct mlx5e_priv *priv = netdev_priv(dev);
struct mlx5e_channels *chs = &priv->channels;
int i;
for (i = 0; i < priv->params.num_channels; i++)
napi_schedule(&priv->channel[i]->napi);
for (i = 0; i < chs->num; i++)
napi_schedule(&chs->c[i]->napi);
}
#endif
......@@ -3627,6 +3728,9 @@ void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
params->rx_cq_moderation.usec =
MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
params->rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
}
u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
......@@ -3641,75 +3745,79 @@ u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
}
static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
struct net_device *netdev,
const struct mlx5e_profile *profile,
void *ppriv)
static void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
u16 max_channels)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
u8 cq_period_mode = 0;
u32 link_speed = 0;
u32 pci_bw = 0;
u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
priv->mdev = mdev;
priv->netdev = netdev;
priv->params.num_channels = profile->max_nch(mdev);
priv->profile = profile;
priv->ppriv = ppriv;
params->num_channels = max_channels;
params->num_tc = 1;
priv->params.lro_timeout =
mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
priv->params.log_sq_size = is_kdump_kernel() ?
/* SQ */
params->log_sq_size = is_kdump_kernel() ?
MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
/* set CQE compression */
priv->params.rx_cqe_compress_def = false;
params->rx_cqe_compress_def = false;
if (MLX5_CAP_GEN(mdev, cqe_compression) &&
MLX5_CAP_GEN(mdev, vport_group_manager)) {
MLX5_CAP_GEN(mdev, vport_group_manager)) {
mlx5e_get_max_linkspeed(mdev, &link_speed);
mlx5e_get_pci_bw(mdev, &pci_bw);
mlx5_core_dbg(mdev, "Max link speed = %d, PCI BW = %d\n",
link_speed, pci_bw);
priv->params.rx_cqe_compress_def =
cqe_compress_heuristic(link_speed, pci_bw);
link_speed, pci_bw);
params->rx_cqe_compress_def = cqe_compress_heuristic(link_speed, pci_bw);
}
MLX5E_SET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS,
priv->params.rx_cqe_compress_def);
mlx5e_set_rq_priv_params(priv);
if (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
priv->params.lro_en = true;
priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
priv->params.tx_cq_moderation.usec =
MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
priv->params.tx_cq_moderation.pkts =
MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
if (priv->params.tx_min_inline_mode == MLX5_INLINE_MODE_NONE &&
MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def);
/* RQ */
mlx5e_set_rq_params(mdev, params);
/* HW LRO */
if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
params->lro_en = true;
params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
/* CQ moderation params */
cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
mlx5e_set_rx_cq_mode_params(params, cq_period_mode);
params->tx_cq_moderation.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
params->tx_cq_moderation.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
/* TX inline */
params->tx_max_inline = mlx5e_get_max_inline_cap(mdev);
mlx5_query_min_inline(mdev, &params->tx_min_inline_mode);
if (params->tx_min_inline_mode == MLX5_INLINE_MODE_NONE &&
!MLX5_CAP_ETH(mdev, wqe_vlan_insert))
priv->params.tx_min_inline_mode = MLX5_INLINE_MODE_L2;
params->tx_min_inline_mode = MLX5_INLINE_MODE_L2;
priv->params.num_tc = 1;
priv->params.rss_hfunc = ETH_RSS_HASH_XOR;
/* RSS */
params->rss_hfunc = ETH_RSS_HASH_XOR;
netdev_rss_key_fill(params->toeplitz_hash_key, sizeof(params->toeplitz_hash_key));
mlx5e_build_default_indir_rqt(mdev, params->indirection_rqt,
MLX5E_INDIR_RQT_SIZE, max_channels);
}
netdev_rss_key_fill(priv->params.toeplitz_hash_key,
sizeof(priv->params.toeplitz_hash_key));
static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
struct net_device *netdev,
const struct mlx5e_profile *profile,
void *ppriv)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
mlx5e_build_default_indir_rqt(mdev, priv->params.indirection_rqt,
MLX5E_INDIR_RQT_SIZE, profile->max_nch(mdev));
priv->mdev = mdev;
priv->netdev = netdev;
priv->profile = profile;
priv->ppriv = ppriv;
/* Initialize pflags */
MLX5E_SET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_BASED_MODER,
priv->params.rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
mlx5e_build_nic_params(mdev, &priv->channels.params, profile->max_nch(mdev));
mutex_init(&priv->state_lock);
......@@ -3795,7 +3903,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
netdev->hw_features |= NETIF_F_RXALL;
netdev->features = netdev->hw_features;
if (!priv->params.lro_en)
if (!priv->channels.params.lro_en)
netdev->features &= ~NETIF_F_LRO;
if (fcs_enabled)
......@@ -3860,8 +3968,8 @@ static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
{
mlx5e_vxlan_cleanup(priv);
if (priv->xdp_prog)
bpf_prog_put(priv->xdp_prog);
if (priv->channels.params.xdp_prog)
bpf_prog_put(priv->channels.params.xdp_prog);
}
static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
......@@ -4024,6 +4132,10 @@ struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
return NULL;
}
#ifdef CONFIG_RFS_ACCEL
netdev->rx_cpu_rmap = mdev->rmap;
#endif
profile->init(mdev, netdev, profile, ppriv);
netif_carrier_off(netdev);
......@@ -4058,7 +4170,7 @@ int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
if (err)
goto out;
err = mlx5e_open_drop_rq(priv);
err = mlx5e_open_drop_rq(mdev, &priv->drop_rq);
if (err) {
mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
goto err_cleanup_tx;
......@@ -4077,7 +4189,7 @@ int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1);
netdev->max_mtu = MLX5E_HW2SW_MTU(max_mtu);
mlx5e_set_dev_port_mtu(netdev);
mlx5e_set_dev_port_mtu(priv);
if (profile->enable)
profile->enable(priv);
......@@ -4091,7 +4203,7 @@ int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
return 0;
err_close_drop_rq:
mlx5e_close_drop_rq(priv);
mlx5e_close_drop_rq(&priv->drop_rq);
err_cleanup_tx:
profile->cleanup_tx(priv);
......@@ -4155,7 +4267,7 @@ void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
mlx5e_destroy_q_counter(priv);
profile->cleanup_rx(priv);
mlx5e_close_drop_rq(priv);
mlx5e_close_drop_rq(&priv->drop_rq);
profile->cleanup_tx(priv);
cancel_delayed_work_sync(&priv->update_stats_work);
}
......
drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
View file @ cc628c96
......@@ -102,14 +102,16 @@ static void mlx5e_rep_update_sw_counters(struct mlx5e_priv *priv)
int i, j;
memset(s, 0, sizeof(*s));
for (i = 0; i < priv->params.num_channels; i++) {
rq_stats = &priv->channel[i]->rq.stats;
for (i = 0; i < priv->channels.num; i++) {
struct mlx5e_channel *c = priv->channels.c[i];
rq_stats = &c->rq.stats;
s->rx_packets += rq_stats->packets;
s->rx_bytes += rq_stats->bytes;
for (j = 0; j < priv->params.num_tc; j++) {
sq_stats = &priv->channel[i]->sq[j].stats;
for (j = 0; j < priv->channels.params.num_tc; j++) {
sq_stats = &c->sq[j].stats;
s->tx_packets += sq_stats->packets;
s->tx_bytes += sq_stats->bytes;
......@@ -187,22 +189,26 @@ int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv)
struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
struct mlx5_eswitch_rep *rep = priv->ppriv;
struct mlx5e_channel *c;
int n, tc, err, num_sqs = 0;
int n, tc, num_sqs = 0;
int err = -ENOMEM;
u16 *sqs;
sqs = kcalloc(priv->params.num_channels * priv->params.num_tc, sizeof(u16), GFP_KERNEL);
sqs = kcalloc(priv->channels.num * priv->channels.params.num_tc, sizeof(u16), GFP_KERNEL);
if (!sqs)
return -ENOMEM;
goto out;
for (n = 0; n < priv->params.num_channels; n++) {
c = priv->channel[n];
for (n = 0; n < priv->channels.num; n++) {
c = priv->channels.c[n];
for (tc = 0; tc < c->num_tc; tc++)
sqs[num_sqs++] = c->sq[tc].sqn;
}
err = mlx5_eswitch_sqs2vport_start(esw, rep, sqs, num_sqs);
kfree(sqs);
out:
if (err)
netdev_warn(priv->netdev, "Failed to add SQs FWD rules %d\n", err);
return err;
}
......@@ -397,42 +403,23 @@ static const struct net_device_ops mlx5e_netdev_ops_rep = {
.ndo_get_offload_stats = mlx5e_get_offload_stats,
};
static void mlx5e_build_rep_netdev_priv(struct mlx5_core_dev *mdev,
struct net_device *netdev,
const struct mlx5e_profile *profile,
void *ppriv)
static void mlx5e_build_rep_params(struct mlx5_core_dev *mdev,
struct mlx5e_params *params)
{
struct mlx5e_priv *priv = netdev_priv(netdev);
u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
priv->params.log_sq_size =
MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
priv->params.rq_wq_type = MLX5_WQ_TYPE_LINKED_LIST;
priv->params.log_rq_size = MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE;
priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
BIT(priv->params.log_rq_size));
priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
params->log_sq_size = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
params->rq_wq_type = MLX5_WQ_TYPE_LINKED_LIST;
params->log_rq_size = MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE;
priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
priv->params.num_tc = 1;
params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
mlx5e_set_rx_cq_mode_params(params, cq_period_mode);
priv->params.lro_wqe_sz =
MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
priv->mdev = mdev;
priv->netdev = netdev;
priv->params.num_channels = profile->max_nch(mdev);
priv->profile = profile;
priv->ppriv = ppriv;
mutex_init(&priv->state_lock);
INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
params->tx_max_inline = mlx5e_get_max_inline_cap(mdev);
params->num_tc = 1;
params->lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
}
static void mlx5e_build_rep_netdev(struct net_device *netdev)
......@@ -458,7 +445,19 @@ static void mlx5e_init_rep(struct mlx5_core_dev *mdev,
const struct mlx5e_profile *profile,
void *ppriv)
{
mlx5e_build_rep_netdev_priv(mdev, netdev, profile, ppriv);
struct mlx5e_priv *priv = netdev_priv(netdev);
priv->mdev = mdev;
priv->netdev = netdev;
priv->profile = profile;
priv->ppriv = ppriv;
mutex_init(&priv->state_lock);
INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
priv->channels.params.num_channels = profile->max_nch(mdev);
mlx5e_build_rep_params(mdev, &priv->channels.params);
mlx5e_build_rep_netdev(netdev);
}
......@@ -503,7 +502,7 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
err_destroy_direct_tirs:
mlx5e_destroy_direct_tirs(priv);
err_destroy_direct_rqts:
for (i = 0; i < priv->params.num_channels; i++)
for (i = 0; i < priv->channels.params.num_channels; i++)
mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
return err;
}
......@@ -516,7 +515,7 @@ static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv)
mlx5e_tc_cleanup(priv);
mlx5_del_flow_rules(rep->vport_rx_rule);
mlx5e_destroy_direct_tirs(priv);
for (i = 0; i < priv->params.num_channels; i++)
for (i = 0; i < priv->channels.params.num_channels; i++)
mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
}
......
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
View file @ cc628c96
......@@ -156,28 +156,6 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
return mlx5e_decompress_cqes_cont(rq, cq, 1, budget_rem) - 1;
}
void mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool val)
{
bool was_opened;
if (!MLX5_CAP_GEN(priv->mdev, cqe_compression))
return;
if (MLX5E_GET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS) == val)
return;
was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
if (was_opened)
mlx5e_close_locked(priv->netdev);
MLX5E_SET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS, val);
mlx5e_set_rq_type_params(priv, priv->params.rq_wq_type);
if (was_opened)
mlx5e_open_locked(priv->netdev);
}
#define RQ_PAGE_SIZE(rq) ((1 << rq->buff.page_order) << PAGE_SHIFT)
static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
......
drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c
View file @ cc628c96
......@@ -293,7 +293,7 @@ void mlx5e_rx_am_work(struct work_struct *work)
struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am);
struct mlx5e_cq_moder cur_profile = profile[am->mode][am->profile_ix];
mlx5_core_modify_cq_moderation(rq->priv->mdev, &rq->cq.mcq,
mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq,
cur_profile.usec, cur_profile.pkts);
am->state = MLX5E_AM_START_MEASURE;
......
drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
View file @ cc628c96
......@@ -236,12 +236,9 @@ static int mlx5e_test_loopback_setup(struct mlx5e_priv *priv,
{
int err = 0;
err = mlx5e_refresh_tirs_self_loopback(priv->mdev, true);
if (err) {
netdev_err(priv->netdev,
"\tFailed to enable UC loopback err(%d)\n", err);
err = mlx5e_refresh_tirs(priv, true);
if (err)
return err;
}
lbtp->loopback_ok = false;
init_completion(&lbtp->comp);
......@@ -258,7 +255,7 @@ static void mlx5e_test_loopback_cleanup(struct mlx5e_priv *priv,
struct mlx5e_lbt_priv *lbtp)
{
dev_remove_pack(&lbtp->pt);
mlx5e_refresh_tirs_self_loopback(priv->mdev, false);
mlx5e_refresh_tirs(priv, false);
}
#define MLX5E_LB_VERIFY_TIMEOUT (msecs_to_jiffies(200))
......
drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
View file @ cc628c96
......@@ -88,6 +88,7 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
{
struct mlx5e_priv *priv = netdev_priv(dev);
int channel_ix = fallback(dev, skb);
u16 num_channels;
int up = 0;
if (!netdev_get_num_tc(dev))
......@@ -99,11 +100,11 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
/* channel_ix can be larger than num_channels since
* dev->num_real_tx_queues = num_channels * num_tc
*/
if (channel_ix >= priv->params.num_channels)
channel_ix = reciprocal_scale(channel_ix,
priv->params.num_channels);
num_channels = priv->channels.params.num_channels;
if (channel_ix >= num_channels)
channel_ix = reciprocal_scale(channel_ix, num_channels);
return priv->channeltc_to_txq_map[channel_ix][up];
return priv->channel_tc2txq[channel_ix][up];
}
static inline int mlx5e_skb_l2_header_offset(struct sk_buff *skb)
......@@ -339,7 +340,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb)
netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct mlx5e_priv *priv = netdev_priv(dev);
struct mlx5e_txqsq *sq = priv->txq_to_sq_map[skb_get_queue_mapping(skb)];
struct mlx5e_txqsq *sq = priv->txq2sq[skb_get_queue_mapping(skb)];
return mlx5e_sq_xmit(sq, skb);
}
......
drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
View file @ cc628c96
......@@ -164,8 +164,7 @@ void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event)
{
struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq);
struct mlx5e_channel *c = cq->channel;
struct mlx5e_priv *priv = c->priv;
struct net_device *netdev = priv->netdev;
struct net_device *netdev = c->netdev;
netdev_err(netdev, "%s: cqn=0x%.6x event=0x%.2x\n",
__func__, mcq->cqn, event);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7