Commit f0313965 authored by Erez Shitrit's avatar Erez Shitrit Committed by Doug Ledford

IB/mlx5: Implement UD QP offloads for IPoIB in the TX flow

In order to support LSO and CSUM in the TX flow the driver does the
following:
* LSO bit for the enum mlx5_ib_qp_flags was added, indicates QP that
  supports LSO offloads.
* Enables the special offload when the QP is created, and enable the
  special work request id (IB_WR_LSO) when comes.
* Calculates the size of the WQE according to the new WQE format that
  support these offloads.
* Handles the new WQE format when arrived, sets the relevant
  fields, and copies the needed data.
Signed-off-by: default avatarErez Shitrit <erezsh@mellanox.com>
Signed-off-by: default avatarEran Ben Elisha <eranbe@mellanox.com>
Reviewed-by: default avatarLeon Romanovsky <leonro@mellanox.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 1015c2e8
...@@ -504,6 +504,11 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, ...@@ -504,6 +504,11 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
(MLX5_CAP_ETH(dev->mdev, csum_cap))) (MLX5_CAP_ETH(dev->mdev, csum_cap)))
props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM; props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
props->device_cap_flags |= IB_DEVICE_UD_TSO;
}
props->vendor_part_id = mdev->pdev->device; props->vendor_part_id = mdev->pdev->device;
props->hw_ver = mdev->pdev->revision; props->hw_ver = mdev->pdev->revision;
......
...@@ -325,11 +325,12 @@ struct mlx5_ib_cq_buf { ...@@ -325,11 +325,12 @@ struct mlx5_ib_cq_buf {
}; };
enum mlx5_ib_qp_flags { enum mlx5_ib_qp_flags {
MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = 1 << 0, MLX5_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO,
MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 1, MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
MLX5_IB_QP_CROSS_CHANNEL = 1 << 2, MLX5_IB_QP_CROSS_CHANNEL = IB_QP_CREATE_CROSS_CHANNEL,
MLX5_IB_QP_MANAGED_SEND = 1 << 3, MLX5_IB_QP_MANAGED_SEND = IB_QP_CREATE_MANAGED_SEND,
MLX5_IB_QP_MANAGED_RECV = 1 << 4, MLX5_IB_QP_MANAGED_RECV = IB_QP_CREATE_MANAGED_RECV,
MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 5,
}; };
struct mlx5_umr_wr { struct mlx5_umr_wr {
......
...@@ -58,6 +58,7 @@ enum { ...@@ -58,6 +58,7 @@ enum {
static const u32 mlx5_ib_opcode[] = { static const u32 mlx5_ib_opcode[] = {
[IB_WR_SEND] = MLX5_OPCODE_SEND, [IB_WR_SEND] = MLX5_OPCODE_SEND,
[IB_WR_LSO] = MLX5_OPCODE_LSO,
[IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM, [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM,
[IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE, [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE,
[IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM, [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM,
...@@ -72,6 +73,9 @@ static const u32 mlx5_ib_opcode[] = { ...@@ -72,6 +73,9 @@ static const u32 mlx5_ib_opcode[] = {
[MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR, [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR,
}; };
struct mlx5_wqe_eth_pad {
u8 rsvd0[16];
};
static int is_qp0(enum ib_qp_type qp_type) static int is_qp0(enum ib_qp_type qp_type)
{ {
...@@ -260,11 +264,11 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap, ...@@ -260,11 +264,11 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
return 0; return 0;
} }
static int sq_overhead(enum ib_qp_type qp_type) static int sq_overhead(struct ib_qp_init_attr *attr)
{ {
int size = 0; int size = 0;
switch (qp_type) { switch (attr->qp_type) {
case IB_QPT_XRC_INI: case IB_QPT_XRC_INI:
size += sizeof(struct mlx5_wqe_xrc_seg); size += sizeof(struct mlx5_wqe_xrc_seg);
/* fall through */ /* fall through */
...@@ -287,6 +291,10 @@ static int sq_overhead(enum ib_qp_type qp_type) ...@@ -287,6 +291,10 @@ static int sq_overhead(enum ib_qp_type qp_type)
break; break;
case IB_QPT_UD: case IB_QPT_UD:
if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
size += sizeof(struct mlx5_wqe_eth_pad) +
sizeof(struct mlx5_wqe_eth_seg);
/* fall through */
case IB_QPT_SMI: case IB_QPT_SMI:
case IB_QPT_GSI: case IB_QPT_GSI:
size += sizeof(struct mlx5_wqe_ctrl_seg) + size += sizeof(struct mlx5_wqe_ctrl_seg) +
...@@ -311,7 +319,7 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr) ...@@ -311,7 +319,7 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr)
int inl_size = 0; int inl_size = 0;
int size; int size;
size = sq_overhead(attr->qp_type); size = sq_overhead(attr);
if (size < 0) if (size < 0)
return size; return size;
...@@ -348,7 +356,7 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, ...@@ -348,7 +356,7 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
return -EINVAL; return -EINVAL;
} }
qp->max_inline_data = wqe_size - sq_overhead(attr->qp_type) - qp->max_inline_data = wqe_size - sq_overhead(attr) -
sizeof(struct mlx5_wqe_inline_seg); sizeof(struct mlx5_wqe_inline_seg);
attr->cap.max_inline_data = qp->max_inline_data; attr->cap.max_inline_data = qp->max_inline_data;
...@@ -783,7 +791,9 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, ...@@ -783,7 +791,9 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
int err; int err;
uuari = &dev->mdev->priv.uuari; uuari = &dev->mdev->priv.uuari;
if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN | IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)) if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN |
IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
IB_QP_CREATE_IPOIB_UD_LSO))
return -EINVAL; return -EINVAL;
if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR) if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
...@@ -1228,6 +1238,14 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd, ...@@ -1228,6 +1238,14 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV) if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV)
qp->flags |= MLX5_IB_QP_MANAGED_RECV; qp->flags |= MLX5_IB_QP_MANAGED_RECV;
} }
if (init_attr->qp_type == IB_QPT_UD &&
(init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO))
if (!MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
mlx5_ib_dbg(dev, "ipoib UD lso qp isn't supported\n");
return -EOPNOTSUPP;
}
if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE; qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
...@@ -1385,6 +1403,13 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd, ...@@ -1385,6 +1403,13 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
/* 0xffffff means we ask to work with cqe version 0 */ /* 0xffffff means we ask to work with cqe version 0 */
MLX5_SET(qpc, qpc, user_index, uidx); MLX5_SET(qpc, qpc, user_index, uidx);
} }
/* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
if (init_attr->qp_type == IB_QPT_UD &&
(init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)) {
qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
qp->flags |= MLX5_IB_QP_LSO;
}
if (init_attr->qp_type == IB_QPT_RAW_PACKET) { if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr; qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;
...@@ -2442,6 +2467,59 @@ static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg, ...@@ -2442,6 +2467,59 @@ static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
rseg->reserved = 0; rseg->reserved = 0;
} }
static void *set_eth_seg(struct mlx5_wqe_eth_seg *eseg,
struct ib_send_wr *wr, void *qend,
struct mlx5_ib_qp *qp, int *size)
{
void *seg = eseg;
memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
if (wr->send_flags & IB_SEND_IP_CSUM)
eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
MLX5_ETH_WQE_L4_CSUM;
seg += sizeof(struct mlx5_wqe_eth_seg);
*size += sizeof(struct mlx5_wqe_eth_seg) / 16;
if (wr->opcode == IB_WR_LSO) {
struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
int size_of_inl_hdr_start = sizeof(eseg->inline_hdr_start);
u64 left, leftlen, copysz;
void *pdata = ud_wr->header;
left = ud_wr->hlen;
eseg->mss = cpu_to_be16(ud_wr->mss);
eseg->inline_hdr_sz = cpu_to_be16(left);
/*
* check if there is space till the end of queue, if yes,
* copy all in one shot, otherwise copy till the end of queue,
* rollback and than the copy the left
*/
leftlen = qend - (void *)eseg->inline_hdr_start;
copysz = min_t(u64, leftlen, left);
memcpy(seg - size_of_inl_hdr_start, pdata, copysz);
if (likely(copysz > size_of_inl_hdr_start)) {
seg += ALIGN(copysz - size_of_inl_hdr_start, 16);
*size += ALIGN(copysz - size_of_inl_hdr_start, 16) / 16;
}
if (unlikely(copysz < left)) { /* the last wqe in the queue */
seg = mlx5_get_send_wqe(qp, 0);
left -= copysz;
pdata += copysz;
memcpy(seg, pdata, left);
seg += ALIGN(left, 16);
*size += ALIGN(left, 16) / 16;
}
}
return seg;
}
static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg, static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
struct ib_send_wr *wr) struct ib_send_wr *wr)
{ {
...@@ -3373,7 +3451,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, ...@@ -3373,7 +3451,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
} }
break; break;
case IB_QPT_UD:
case IB_QPT_SMI: case IB_QPT_SMI:
case IB_QPT_GSI: case IB_QPT_GSI:
set_datagram_seg(seg, wr); set_datagram_seg(seg, wr);
...@@ -3382,7 +3459,29 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, ...@@ -3382,7 +3459,29 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
if (unlikely((seg == qend))) if (unlikely((seg == qend)))
seg = mlx5_get_send_wqe(qp, 0); seg = mlx5_get_send_wqe(qp, 0);
break; break;
case IB_QPT_UD:
set_datagram_seg(seg, wr);
seg += sizeof(struct mlx5_wqe_datagram_seg);
size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
if (unlikely((seg == qend)))
seg = mlx5_get_send_wqe(qp, 0);
/* handle qp that supports ud offload */
if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
struct mlx5_wqe_eth_pad *pad;
pad = seg;
memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
seg += sizeof(struct mlx5_wqe_eth_pad);
size += sizeof(struct mlx5_wqe_eth_pad) / 16;
seg = set_eth_seg(seg, wr, qend, qp, &size);
if (unlikely((seg == qend)))
seg = mlx5_get_send_wqe(qp, 0);
}
break;
case MLX5_IB_QPT_REG_UMR: case MLX5_IB_QPT_REG_UMR:
if (wr->opcode != MLX5_IB_WR_UMR) { if (wr->opcode != MLX5_IB_WR_UMR) {
err = -EINVAL; err = -EINVAL;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment