Commit 8ef0c040 authored by David S. Miller's avatar David S. Miller

Merge branch 'mlxsw-Update-main-pool-computation-and-pool-size-limits'

Ido Schimmel says:

====================
mlxsw: Update main pool computation and pool size limits

Petr says:

In Spectrum ASICs, the shared buffer is an area of memory where packets are
kept until they can be transmitted. There are two resources associated with
shared buffer size: cap_total_buffer_size and cap_guaranteed_shared_buffer.
So far, mlxsw has been using the former as a limit when validating shared
buffer pool size configuration. However, the total size also includes
headrooms and reserved space, which really cannot be used for shared buffer
pools. Patch #1 mends this and has mlxsw use the guaranteed size.

To configure default pool sizes, mlxsw has historically hard-coded one or
two smallish pools, and one "main" pool that took most of the shared buffer
(that would be pool 0 on ingress and pool 4 on egress). During the
development of Spectrum-2, it became clear that the shared buffer size
keeps shrinking as bugs are identified and worked around. In order to
prevent having to tweak the size of pools 0 and 4 to catch up with updates
to values reported by the FW, patch #2 changes the way these pools are set.
Instead of hard-coding a fixed value, the main pool now takes whatever is
left from the guaranteed size after the smaller pool(s) are taken into
account.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 06cd9da5 bc9f6e94
...@@ -26,7 +26,7 @@ enum mlxsw_res_id { ...@@ -26,7 +26,7 @@ enum mlxsw_res_id {
MLXSW_RES_ID_MAX_LAG_MEMBERS, MLXSW_RES_ID_MAX_LAG_MEMBERS,
MLXSW_RES_ID_LOCAL_PORTS_IN_1X, MLXSW_RES_ID_LOCAL_PORTS_IN_1X,
MLXSW_RES_ID_LOCAL_PORTS_IN_2X, MLXSW_RES_ID_LOCAL_PORTS_IN_2X,
MLXSW_RES_ID_MAX_BUFFER_SIZE, MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER,
MLXSW_RES_ID_CELL_SIZE, MLXSW_RES_ID_CELL_SIZE,
MLXSW_RES_ID_MAX_HEADROOM_SIZE, MLXSW_RES_ID_MAX_HEADROOM_SIZE,
MLXSW_RES_ID_ACL_MAX_TCAM_REGIONS, MLXSW_RES_ID_ACL_MAX_TCAM_REGIONS,
...@@ -82,7 +82,7 @@ static u16 mlxsw_res_ids[] = { ...@@ -82,7 +82,7 @@ static u16 mlxsw_res_ids[] = {
[MLXSW_RES_ID_MAX_LAG_MEMBERS] = 0x2521, [MLXSW_RES_ID_MAX_LAG_MEMBERS] = 0x2521,
[MLXSW_RES_ID_LOCAL_PORTS_IN_1X] = 0x2610, [MLXSW_RES_ID_LOCAL_PORTS_IN_1X] = 0x2610,
[MLXSW_RES_ID_LOCAL_PORTS_IN_2X] = 0x2611, [MLXSW_RES_ID_LOCAL_PORTS_IN_2X] = 0x2611,
[MLXSW_RES_ID_MAX_BUFFER_SIZE] = 0x2802, /* Bytes */ [MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER] = 0x2805, /* Bytes */
[MLXSW_RES_ID_CELL_SIZE] = 0x2803, /* Bytes */ [MLXSW_RES_ID_CELL_SIZE] = 0x2803, /* Bytes */
[MLXSW_RES_ID_MAX_HEADROOM_SIZE] = 0x2811, /* Bytes */ [MLXSW_RES_ID_MAX_HEADROOM_SIZE] = 0x2811, /* Bytes */
[MLXSW_RES_ID_ACL_MAX_TCAM_REGIONS] = 0x2901, [MLXSW_RES_ID_ACL_MAX_TCAM_REGIONS] = 0x2901,
......
...@@ -35,6 +35,7 @@ struct mlxsw_sp_sb_cm { ...@@ -35,6 +35,7 @@ struct mlxsw_sp_sb_cm {
}; };
#define MLXSW_SP_SB_INFI -1U #define MLXSW_SP_SB_INFI -1U
#define MLXSW_SP_SB_REST -2U
struct mlxsw_sp_sb_pm { struct mlxsw_sp_sb_pm {
u32 min_buff; u32 min_buff;
...@@ -421,19 +422,16 @@ static void mlxsw_sp_sb_ports_fini(struct mlxsw_sp *mlxsw_sp) ...@@ -421,19 +422,16 @@ static void mlxsw_sp_sb_ports_fini(struct mlxsw_sp *mlxsw_sp)
.freeze_size = _freeze_size, \ .freeze_size = _freeze_size, \
} }
#define MLXSW_SP1_SB_PR_INGRESS_SIZE 12440000
#define MLXSW_SP1_SB_PR_EGRESS_SIZE 13232000
#define MLXSW_SP1_SB_PR_CPU_SIZE (256 * 1000) #define MLXSW_SP1_SB_PR_CPU_SIZE (256 * 1000)
/* Order according to mlxsw_sp1_sb_pool_dess */ /* Order according to mlxsw_sp1_sb_pool_dess */
static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = { static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = {
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),
MLXSW_SP1_SB_PR_INGRESS_SIZE),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,
MLXSW_SP1_SB_PR_EGRESS_SIZE, true, false), true, false),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
...@@ -445,19 +443,16 @@ static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = { ...@@ -445,19 +443,16 @@ static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = {
MLXSW_SP1_SB_PR_CPU_SIZE, true, false), MLXSW_SP1_SB_PR_CPU_SIZE, true, false),
}; };
#define MLXSW_SP2_SB_PR_INGRESS_SIZE 35297568
#define MLXSW_SP2_SB_PR_EGRESS_SIZE 35297568
#define MLXSW_SP2_SB_PR_CPU_SIZE (256 * 1000) #define MLXSW_SP2_SB_PR_CPU_SIZE (256 * 1000)
/* Order according to mlxsw_sp2_sb_pool_dess */ /* Order according to mlxsw_sp2_sb_pool_dess */
static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = { static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = {
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),
MLXSW_SP2_SB_PR_INGRESS_SIZE),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,
MLXSW_SP2_SB_PR_EGRESS_SIZE, true, false), true, false),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
...@@ -471,11 +466,33 @@ static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = { ...@@ -471,11 +466,33 @@ static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = {
static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp, static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp,
const struct mlxsw_sp_sb_pr *prs, const struct mlxsw_sp_sb_pr *prs,
const struct mlxsw_sp_sb_pool_des *pool_dess,
size_t prs_len) size_t prs_len)
{ {
/* Round down, unlike mlxsw_sp_bytes_cells(). */
u32 sb_cells = mlxsw_sp->sb->sb_size / mlxsw_sp->sb->cell_size;
u32 rest_cells[2] = {sb_cells, sb_cells};
int i; int i;
int err; int err;
/* Calculate how much space to give to the "REST" pools in either
* direction.
*/
for (i = 0; i < prs_len; i++) {
enum mlxsw_reg_sbxx_dir dir = pool_dess[i].dir;
u32 size = prs[i].size;
u32 size_cells;
if (size == MLXSW_SP_SB_INFI || size == MLXSW_SP_SB_REST)
continue;
size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);
if (WARN_ON_ONCE(size_cells > rest_cells[dir]))
continue;
rest_cells[dir] -= size_cells;
}
for (i = 0; i < prs_len; i++) { for (i = 0; i < prs_len; i++) {
u32 size = prs[i].size; u32 size = prs[i].size;
u32 size_cells; u32 size_cells;
...@@ -483,6 +500,10 @@ static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp, ...@@ -483,6 +500,10 @@ static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp,
if (size == MLXSW_SP_SB_INFI) { if (size == MLXSW_SP_SB_INFI) {
err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode, err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
0, true); 0, true);
} else if (size == MLXSW_SP_SB_REST) {
size_cells = rest_cells[pool_dess[i].dir];
err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
size_cells, false);
} else { } else {
size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size); size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);
err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode, err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
...@@ -904,7 +925,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) ...@@ -904,7 +925,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp)
if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, CELL_SIZE)) if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, CELL_SIZE))
return -EIO; return -EIO;
if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_BUFFER_SIZE)) if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, GUARANTEED_SHARED_BUFFER))
return -EIO; return -EIO;
if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_HEADROOM_SIZE)) if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_HEADROOM_SIZE))
...@@ -915,7 +936,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) ...@@ -915,7 +936,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp)
return -ENOMEM; return -ENOMEM;
mlxsw_sp->sb->cell_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, CELL_SIZE); mlxsw_sp->sb->cell_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, CELL_SIZE);
mlxsw_sp->sb->sb_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, mlxsw_sp->sb->sb_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
MAX_BUFFER_SIZE); GUARANTEED_SHARED_BUFFER);
max_headroom_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, max_headroom_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
MAX_HEADROOM_SIZE); MAX_HEADROOM_SIZE);
/* Round down, because this limit must not be overstepped. */ /* Round down, because this limit must not be overstepped. */
...@@ -926,6 +947,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) ...@@ -926,6 +947,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp)
if (err) if (err)
goto err_sb_ports_init; goto err_sb_ports_init;
err = mlxsw_sp_sb_prs_init(mlxsw_sp, mlxsw_sp->sb_vals->prs, err = mlxsw_sp_sb_prs_init(mlxsw_sp, mlxsw_sp->sb_vals->prs,
mlxsw_sp->sb_vals->pool_dess,
mlxsw_sp->sb_vals->pool_count); mlxsw_sp->sb_vals->pool_count);
if (err) if (err)
goto err_sb_prs_init; goto err_sb_prs_init;
...@@ -1013,7 +1035,8 @@ int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core, ...@@ -1013,7 +1035,8 @@ int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core,
mode = (enum mlxsw_reg_sbpr_mode) threshold_type; mode = (enum mlxsw_reg_sbpr_mode) threshold_type;
pr = &mlxsw_sp->sb_vals->prs[pool_index]; pr = &mlxsw_sp->sb_vals->prs[pool_index];
if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_BUFFER_SIZE)) { if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core,
GUARANTEED_SHARED_BUFFER)) {
NL_SET_ERR_MSG_MOD(extack, "Exceeded shared buffer size"); NL_SET_ERR_MSG_MOD(extack, "Exceeded shared buffer size");
return -EINVAL; return -EINVAL;
} }
......
...@@ -305,7 +305,8 @@ mlxsw_sp_qdisc_red_check_params(struct mlxsw_sp_port *mlxsw_sp_port, ...@@ -305,7 +305,8 @@ mlxsw_sp_qdisc_red_check_params(struct mlxsw_sp_port *mlxsw_sp_port,
p->max); p->max);
return -EINVAL; return -EINVAL;
} }
if (p->max > MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_BUFFER_SIZE)) { if (p->max > MLXSW_CORE_RES_GET(mlxsw_sp->core,
GUARANTEED_SHARED_BUFFER)) {
dev_err(mlxsw_sp->bus_info->dev, dev_err(mlxsw_sp->bus_info->dev,
"spectrum: RED: max value %u is too big\n", p->max); "spectrum: RED: max value %u is too big\n", p->max);
return -EINVAL; return -EINVAL;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment