Commit 25361e02 authored by Haggai Eran's avatar Haggai Eran Committed by Doug Ledford

IB/mlx5: Generate completions in software

The GSI QP emulation requires also emulating completions for transmitted
MADs. The CQ on which these completions are generated can also be used by
the hardware, and the MAD layer is free to use any CQ of the device for the
GSI QP.

Add a method for generating software completions to each mlx5 CQ. Software
completions are polled first, and generate calls to the completion handler
callback if necessary.
Reviewed-by: default avatarLeon Romanovsky <leonro@mellanox.com>
Signed-off-by: default avatarHaggai Eran <haggaie@mellanox.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 7722f47e
...@@ -568,18 +568,44 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq, ...@@ -568,18 +568,44 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
return 0; return 0;
} }
static int poll_soft_wc(struct mlx5_ib_cq *cq, int num_entries,
struct ib_wc *wc)
{
struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
struct mlx5_ib_wc *soft_wc, *next;
int npolled = 0;
list_for_each_entry_safe(soft_wc, next, &cq->wc_list, list) {
if (npolled >= num_entries)
break;
mlx5_ib_dbg(dev, "polled software generated completion on CQ 0x%x\n",
cq->mcq.cqn);
wc[npolled++] = soft_wc->wc;
list_del(&soft_wc->list);
kfree(soft_wc);
}
return npolled;
}
int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
{ {
struct mlx5_ib_cq *cq = to_mcq(ibcq); struct mlx5_ib_cq *cq = to_mcq(ibcq);
struct mlx5_ib_qp *cur_qp = NULL; struct mlx5_ib_qp *cur_qp = NULL;
unsigned long flags; unsigned long flags;
int soft_polled = 0;
int npolled; int npolled;
int err = 0; int err = 0;
spin_lock_irqsave(&cq->lock, flags); spin_lock_irqsave(&cq->lock, flags);
for (npolled = 0; npolled < num_entries; npolled++) { if (unlikely(!list_empty(&cq->wc_list)))
err = mlx5_poll_one(cq, &cur_qp, wc + npolled); soft_polled = poll_soft_wc(cq, num_entries, wc);
for (npolled = 0; npolled < num_entries - soft_polled; npolled++) {
err = mlx5_poll_one(cq, &cur_qp, wc + soft_polled + npolled);
if (err) if (err)
break; break;
} }
...@@ -590,7 +616,7 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) ...@@ -590,7 +616,7 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
spin_unlock_irqrestore(&cq->lock, flags); spin_unlock_irqrestore(&cq->lock, flags);
if (err == 0 || err == -EAGAIN) if (err == 0 || err == -EAGAIN)
return npolled; return soft_polled + npolled;
else else
return err; return err;
} }
...@@ -598,16 +624,27 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) ...@@ -598,16 +624,27 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
{ {
struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev; struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;
struct mlx5_ib_cq *cq = to_mcq(ibcq);
void __iomem *uar_page = mdev->priv.uuari.uars[0].map; void __iomem *uar_page = mdev->priv.uuari.uars[0].map;
unsigned long irq_flags;
int ret = 0;
spin_lock_irqsave(&cq->lock, irq_flags);
if (cq->notify_flags != IB_CQ_NEXT_COMP)
cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
mlx5_cq_arm(&to_mcq(ibcq)->mcq, if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !list_empty(&cq->wc_list))
ret = 1;
spin_unlock_irqrestore(&cq->lock, irq_flags);
mlx5_cq_arm(&cq->mcq,
(flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT, MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
uar_page, uar_page,
MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock), MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock),
to_mcq(ibcq)->mcq.cons_index); to_mcq(ibcq)->mcq.cons_index);
return 0; return ret;
} }
static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf, static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf,
...@@ -760,6 +797,14 @@ static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq) ...@@ -760,6 +797,14 @@ static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
mlx5_db_free(dev->mdev, &cq->db); mlx5_db_free(dev->mdev, &cq->db);
} }
static void notify_soft_wc_handler(struct work_struct *work)
{
struct mlx5_ib_cq *cq = container_of(work, struct mlx5_ib_cq,
notify_work);
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr, const struct ib_cq_init_attr *attr,
struct ib_ucontext *context, struct ib_ucontext *context,
...@@ -810,6 +855,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, ...@@ -810,6 +855,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
&index, &inlen); &index, &inlen);
if (err) if (err)
goto err_create; goto err_create;
INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
} }
cq->cqe_size = cqe_size; cq->cqe_size = cqe_size;
...@@ -835,6 +882,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, ...@@ -835,6 +882,8 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
cq->mcq.comp = mlx5_ib_cq_comp; cq->mcq.comp = mlx5_ib_cq_comp;
cq->mcq.event = mlx5_ib_cq_event; cq->mcq.event = mlx5_ib_cq_event;
INIT_LIST_HEAD(&cq->wc_list);
if (context) if (context)
if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) { if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
err = -EFAULT; err = -EFAULT;
...@@ -1222,3 +1271,27 @@ int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq) ...@@ -1222,3 +1271,27 @@ int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq)
cq = to_mcq(ibcq); cq = to_mcq(ibcq);
return cq->cqe_size; return cq->cqe_size;
} }
/* Called from atomic context */
int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc)
{
struct mlx5_ib_wc *soft_wc;
struct mlx5_ib_cq *cq = to_mcq(ibcq);
unsigned long flags;
soft_wc = kmalloc(sizeof(*soft_wc), GFP_ATOMIC);
if (!soft_wc)
return -ENOMEM;
soft_wc->wc = *wc;
spin_lock_irqsave(&cq->lock, flags);
list_add_tail(&soft_wc->list, &cq->wc_list);
if (cq->notify_flags == IB_CQ_NEXT_COMP ||
wc->status != IB_WC_SUCCESS) {
cq->notify_flags = 0;
schedule_work(&cq->notify_work);
}
spin_unlock_irqrestore(&cq->lock, flags);
return 0;
}
...@@ -393,6 +393,14 @@ struct mlx5_ib_cq { ...@@ -393,6 +393,14 @@ struct mlx5_ib_cq {
struct ib_umem *resize_umem; struct ib_umem *resize_umem;
int cqe_size; int cqe_size;
u32 create_flags; u32 create_flags;
struct list_head wc_list;
enum ib_cq_notify_flags notify_flags;
struct work_struct notify_work;
};
struct mlx5_ib_wc {
struct ib_wc wc;
struct list_head list;
}; };
struct mlx5_ib_srq { struct mlx5_ib_srq {
...@@ -785,6 +793,8 @@ int mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr, ...@@ -785,6 +793,8 @@ int mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr,
struct ib_recv_wr **bad_wr); struct ib_recv_wr **bad_wr);
void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi); void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
static inline void init_query_mad(struct ib_smp *mad) static inline void init_query_mad(struct ib_smp *mad)
{ {
mad->base_version = 1; mad->base_version = 1;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment