Commit d7037ad7 authored by Tariq Toukan's avatar Tariq Toukan Committed by Saeed Mahameed

net/mlx5: Fix QP fragmented buffer allocation

Fix bad alignment of SQ buffer in fragmented QP allocation.
It should start directly after RQ buffer ends.

Take special care of the end case where the RQ buffer does not occupy
a whole page. RQ size is a power of two, so would be the case only for
small RQ sizes (RQ size < PAGE_SIZE).

Fix wrong assignments for sqb->size (mistakenly assigned RQ size),
and for npages value of RQ and SQ.

Fixes: 3a2f7033 ("net/mlx5: Use order-0 allocations for all WQ types")
Signed-off-by: default avatarTariq Toukan <tariqt@mellanox.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
parent 8c49f54a
...@@ -123,7 +123,7 @@ int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, ...@@ -123,7 +123,7 @@ int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size,
int i; int i;
buf->size = size; buf->size = size;
buf->npages = 1 << get_order(size); buf->npages = DIV_ROUND_UP(size, PAGE_SIZE);
buf->page_shift = PAGE_SHIFT; buf->page_shift = PAGE_SHIFT;
buf->frags = kcalloc(buf->npages, sizeof(struct mlx5_buf_list), buf->frags = kcalloc(buf->npages, sizeof(struct mlx5_buf_list),
GFP_KERNEL); GFP_KERNEL);
......
...@@ -113,35 +113,45 @@ int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, ...@@ -113,35 +113,45 @@ int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
return err; return err;
} }
static void mlx5e_qp_set_frag_buf(struct mlx5_frag_buf *buf, static void mlx5_qp_set_frag_buf(struct mlx5_frag_buf *buf,
struct mlx5_wq_qp *qp) struct mlx5_wq_qp *qp)
{ {
struct mlx5_frag_buf_ctrl *sq_fbc;
struct mlx5_frag_buf *rqb, *sqb; struct mlx5_frag_buf *rqb, *sqb;
rqb = &qp->rq.fbc.frag_buf; rqb = &qp->rq.fbc.frag_buf;
*rqb = *buf; *rqb = *buf;
rqb->size = mlx5_wq_cyc_get_byte_size(&qp->rq); rqb->size = mlx5_wq_cyc_get_byte_size(&qp->rq);
rqb->npages = 1 << get_order(rqb->size); rqb->npages = DIV_ROUND_UP(rqb->size, PAGE_SIZE);
sqb = &qp->sq.fbc.frag_buf; sq_fbc = &qp->sq.fbc;
*sqb = *buf; sqb = &sq_fbc->frag_buf;
sqb->size = mlx5_wq_cyc_get_byte_size(&qp->rq); *sqb = *buf;
sqb->npages = 1 << get_order(sqb->size); sqb->size = mlx5_wq_cyc_get_byte_size(&qp->sq);
sqb->npages = DIV_ROUND_UP(sqb->size, PAGE_SIZE);
sqb->frags += rqb->npages; /* first part is for the rq */ sqb->frags += rqb->npages; /* first part is for the rq */
if (sq_fbc->strides_offset)
sqb->frags--;
} }
int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
void *qpc, struct mlx5_wq_qp *wq, void *qpc, struct mlx5_wq_qp *wq,
struct mlx5_wq_ctrl *wq_ctrl) struct mlx5_wq_ctrl *wq_ctrl)
{ {
u32 sq_strides_offset;
int err; int err;
mlx5_fill_fbc(MLX5_GET(qpc, qpc, log_rq_stride) + 4, mlx5_fill_fbc(MLX5_GET(qpc, qpc, log_rq_stride) + 4,
MLX5_GET(qpc, qpc, log_rq_size), MLX5_GET(qpc, qpc, log_rq_size),
&wq->rq.fbc); &wq->rq.fbc);
mlx5_fill_fbc(ilog2(MLX5_SEND_WQE_BB),
MLX5_GET(qpc, qpc, log_sq_size), sq_strides_offset =
&wq->sq.fbc); ((wq->rq.fbc.frag_sz_m1 + 1) % PAGE_SIZE) / MLX5_SEND_WQE_BB;
mlx5_fill_fbc_offset(ilog2(MLX5_SEND_WQE_BB),
MLX5_GET(qpc, qpc, log_sq_size),
sq_strides_offset,
&wq->sq.fbc);
err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node); err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
if (err) { if (err) {
...@@ -156,7 +166,7 @@ int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, ...@@ -156,7 +166,7 @@ int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
goto err_db_free; goto err_db_free;
} }
mlx5e_qp_set_frag_buf(&wq_ctrl->buf, wq); mlx5_qp_set_frag_buf(&wq_ctrl->buf, wq);
wq->rq.db = &wq_ctrl->db.db[MLX5_RCV_DBR]; wq->rq.db = &wq_ctrl->db.db[MLX5_RCV_DBR];
wq->sq.db = &wq_ctrl->db.db[MLX5_SND_DBR]; wq->sq.db = &wq_ctrl->db.db[MLX5_SND_DBR];
......
...@@ -358,6 +358,7 @@ struct mlx5_frag_buf_ctrl { ...@@ -358,6 +358,7 @@ struct mlx5_frag_buf_ctrl {
struct mlx5_frag_buf frag_buf; struct mlx5_frag_buf frag_buf;
u32 sz_m1; u32 sz_m1;
u32 frag_sz_m1; u32 frag_sz_m1;
u32 strides_offset;
u8 log_sz; u8 log_sz;
u8 log_stride; u8 log_stride;
u8 log_frag_strides; u8 log_frag_strides;
...@@ -983,14 +984,22 @@ static inline u32 mlx5_base_mkey(const u32 key) ...@@ -983,14 +984,22 @@ static inline u32 mlx5_base_mkey(const u32 key)
return key & 0xffffff00u; return key & 0xffffff00u;
} }
static inline void mlx5_fill_fbc(u8 log_stride, u8 log_sz, static inline void mlx5_fill_fbc_offset(u8 log_stride, u8 log_sz,
struct mlx5_frag_buf_ctrl *fbc) u32 strides_offset,
struct mlx5_frag_buf_ctrl *fbc)
{ {
fbc->log_stride = log_stride; fbc->log_stride = log_stride;
fbc->log_sz = log_sz; fbc->log_sz = log_sz;
fbc->sz_m1 = (1 << fbc->log_sz) - 1; fbc->sz_m1 = (1 << fbc->log_sz) - 1;
fbc->log_frag_strides = PAGE_SHIFT - fbc->log_stride; fbc->log_frag_strides = PAGE_SHIFT - fbc->log_stride;
fbc->frag_sz_m1 = (1 << fbc->log_frag_strides) - 1; fbc->frag_sz_m1 = (1 << fbc->log_frag_strides) - 1;
fbc->strides_offset = strides_offset;
}
static inline void mlx5_fill_fbc(u8 log_stride, u8 log_sz,
struct mlx5_frag_buf_ctrl *fbc)
{
mlx5_fill_fbc_offset(log_stride, log_sz, 0, fbc);
} }
static inline void mlx5_core_init_cq_frag_buf(struct mlx5_frag_buf_ctrl *fbc, static inline void mlx5_core_init_cq_frag_buf(struct mlx5_frag_buf_ctrl *fbc,
...@@ -1004,7 +1013,10 @@ static inline void mlx5_core_init_cq_frag_buf(struct mlx5_frag_buf_ctrl *fbc, ...@@ -1004,7 +1013,10 @@ static inline void mlx5_core_init_cq_frag_buf(struct mlx5_frag_buf_ctrl *fbc,
static inline void *mlx5_frag_buf_get_wqe(struct mlx5_frag_buf_ctrl *fbc, static inline void *mlx5_frag_buf_get_wqe(struct mlx5_frag_buf_ctrl *fbc,
u32 ix) u32 ix)
{ {
unsigned int frag = (ix >> fbc->log_frag_strides); unsigned int frag;
ix += fbc->strides_offset;
frag = ix >> fbc->log_frag_strides;
return fbc->frag_buf.frags[frag].buf + return fbc->frag_buf.frags[frag].buf +
((fbc->frag_sz_m1 & ix) << fbc->log_stride); ((fbc->frag_sz_m1 & ix) << fbc->log_stride);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment