Commit e088a685 authored by Yixian Liu's avatar Yixian Liu Committed by Doug Ledford

RDMA/hns: Support rq record doorbell for the user space

This patch adds interfaces and definitions to support the rq record
doorbell for the user space.
Signed-off-by: default avatarYixian Liu <liuyixian@huawei.com>
Signed-off-by: default avatarLijun Ou <oulijun@huawei.com>
Signed-off-by: default avatarWei Hu (Xavier) <xavier.huwei@huawei.com>
Signed-off-by: default avatarShaobo Xu <xushaobo2@huawei.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 036ef0a1
...@@ -7,7 +7,7 @@ ccflags-y := -Idrivers/net/ethernet/hisilicon/hns3 ...@@ -7,7 +7,7 @@ ccflags-y := -Idrivers/net/ethernet/hisilicon/hns3
obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o
hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \ hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
hns_roce_cq.o hns_roce_alloc.o hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o
obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o
hns-roce-hw-v1-objs := hns_roce_hw_v1.o hns-roce-hw-v1-objs := hns_roce_hw_v1.o
obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o
......
/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
/*
* Copyright (c) 2017 Hisilicon Limited.
* Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
*/
#include <linux/platform_device.h>
#include <rdma/ib_umem.h>
#include "hns_roce_device.h"
int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
struct hns_roce_db *db)
{
struct hns_roce_user_db_page *page;
int ret = 0;
mutex_lock(&context->page_mutex);
list_for_each_entry(page, &context->page_list, list)
if (page->user_virt == (virt & PAGE_MASK))
goto found;
page = kmalloc(sizeof(*page), GFP_KERNEL);
if (!page) {
ret = -ENOMEM;
goto out;
}
refcount_set(&page->refcount, 1);
page->user_virt = (virt & PAGE_MASK);
page->umem = ib_umem_get(&context->ibucontext, virt & PAGE_MASK,
PAGE_SIZE, 0, 0);
if (IS_ERR(page->umem)) {
ret = PTR_ERR(page->umem);
kfree(page);
goto out;
}
list_add(&page->list, &context->page_list);
found:
db->dma = sg_dma_address(page->umem->sg_head.sgl) +
(virt & ~PAGE_MASK);
db->u.user_page = page;
refcount_inc(&page->refcount);
out:
mutex_unlock(&context->page_mutex);
return ret;
}
EXPORT_SYMBOL(hns_roce_db_map_user);
void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
struct hns_roce_db *db)
{
mutex_lock(&context->page_mutex);
refcount_dec(&db->u.user_page->refcount);
if (refcount_dec_if_one(&db->u.user_page->refcount)) {
list_del(&db->u.user_page->list);
ib_umem_release(db->u.user_page->umem);
kfree(db->u.user_page);
}
mutex_unlock(&context->page_mutex);
}
EXPORT_SYMBOL(hns_roce_db_unmap_user);
...@@ -105,6 +105,10 @@ ...@@ -105,6 +105,10 @@
#define PAGES_SHIFT_24 24 #define PAGES_SHIFT_24 24
#define PAGES_SHIFT_32 32 #define PAGES_SHIFT_32 32
enum {
HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 << 0,
};
enum hns_roce_qp_state { enum hns_roce_qp_state {
HNS_ROCE_QP_STATE_RST, HNS_ROCE_QP_STATE_RST,
HNS_ROCE_QP_STATE_INIT, HNS_ROCE_QP_STATE_INIT,
...@@ -178,7 +182,8 @@ enum { ...@@ -178,7 +182,8 @@ enum {
enum { enum {
HNS_ROCE_CAP_FLAG_REREG_MR = BIT(0), HNS_ROCE_CAP_FLAG_REREG_MR = BIT(0),
HNS_ROCE_CAP_FLAG_ROCE_V1_V2 = BIT(1), HNS_ROCE_CAP_FLAG_ROCE_V1_V2 = BIT(1),
HNS_ROCE_CAP_FLAG_RQ_INLINE = BIT(2) HNS_ROCE_CAP_FLAG_RQ_INLINE = BIT(2),
HNS_ROCE_CAP_FLAG_RECORD_DB = BIT(3)
}; };
enum hns_roce_mtt_type { enum hns_roce_mtt_type {
...@@ -186,6 +191,10 @@ enum hns_roce_mtt_type { ...@@ -186,6 +191,10 @@ enum hns_roce_mtt_type {
MTT_TYPE_CQE, MTT_TYPE_CQE,
}; };
enum {
HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4
};
#define HNS_ROCE_CMD_SUCCESS 1 #define HNS_ROCE_CMD_SUCCESS 1
#define HNS_ROCE_PORT_DOWN 0 #define HNS_ROCE_PORT_DOWN 0
...@@ -203,6 +212,8 @@ struct hns_roce_uar { ...@@ -203,6 +212,8 @@ struct hns_roce_uar {
struct hns_roce_ucontext { struct hns_roce_ucontext {
struct ib_ucontext ibucontext; struct ib_ucontext ibucontext;
struct hns_roce_uar uar; struct hns_roce_uar uar;
struct list_head page_list;
struct mutex page_mutex;
}; };
struct hns_roce_pd { struct hns_roce_pd {
...@@ -335,6 +346,33 @@ struct hns_roce_buf { ...@@ -335,6 +346,33 @@ struct hns_roce_buf {
int page_shift; int page_shift;
}; };
struct hns_roce_db_pgdir {
struct list_head list;
DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE);
DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / 2);
unsigned long *bits[2];
u32 *page;
dma_addr_t db_dma;
};
struct hns_roce_user_db_page {
struct list_head list;
struct ib_umem *umem;
unsigned long user_virt;
refcount_t refcount;
};
struct hns_roce_db {
u32 *db_record;
union {
struct hns_roce_db_pgdir *pgdir;
struct hns_roce_user_db_page *user_page;
} u;
dma_addr_t dma;
int index;
int order;
};
struct hns_roce_cq_buf { struct hns_roce_cq_buf {
struct hns_roce_buf hr_buf; struct hns_roce_buf hr_buf;
struct hns_roce_mtt hr_mtt; struct hns_roce_mtt hr_mtt;
...@@ -466,6 +504,8 @@ struct hns_roce_qp { ...@@ -466,6 +504,8 @@ struct hns_roce_qp {
struct ib_qp ibqp; struct ib_qp ibqp;
struct hns_roce_buf hr_buf; struct hns_roce_buf hr_buf;
struct hns_roce_wq rq; struct hns_roce_wq rq;
struct hns_roce_db rdb;
u8 rdb_en;
u32 doorbell_qpn; u32 doorbell_qpn;
__le32 sq_signal_bits; __le32 sq_signal_bits;
u32 sq_next_wqe; u32 sq_next_wqe;
...@@ -930,6 +970,10 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, ...@@ -930,6 +970,10 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq); int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq);
void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq); void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq);
int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
struct hns_roce_db *db);
void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
struct hns_roce_db *db);
void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn); void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);
void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type); void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);
void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type); void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);
......
...@@ -1168,7 +1168,8 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) ...@@ -1168,7 +1168,8 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR |
HNS_ROCE_CAP_FLAG_ROCE_V1_V2 | HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
HNS_ROCE_CAP_FLAG_RQ_INLINE; HNS_ROCE_CAP_FLAG_RQ_INLINE |
HNS_ROCE_CAP_FLAG_RECORD_DB;
caps->pkey_table_len[0] = 1; caps->pkey_table_len[0] = 1;
caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM;
...@@ -2274,6 +2275,23 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, ...@@ -2274,6 +2275,23 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp,
hr_qp->qkey = attr->qkey; hr_qp->qkey = attr->qkey;
} }
if (hr_qp->rdb_en) {
roce_set_bit(context->byte_68_rq_db,
V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
roce_set_bit(qpc_mask->byte_68_rq_db,
V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
}
roce_set_field(context->byte_68_rq_db,
V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
((u32)hr_qp->rdb.dma) >> 1);
roce_set_field(qpc_mask->byte_68_rq_db,
V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
context->rq_db_record_addr = hr_qp->rdb.dma >> 32;
qpc_mask->rq_db_record_addr = 0;
roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 1); roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 1);
roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0); roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
...@@ -3211,6 +3229,8 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, ...@@ -3211,6 +3229,8 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
hr_qp->sq.tail = 0; hr_qp->sq.tail = 0;
hr_qp->sq_next_wqe = 0; hr_qp->sq_next_wqe = 0;
hr_qp->next_sge = 0; hr_qp->next_sge = 0;
if (hr_qp->rq.wqe_cnt)
*hr_qp->rdb.db_record = 0;
} }
out: out:
...@@ -3437,6 +3457,10 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, ...@@ -3437,6 +3457,10 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
if (is_user) { if (is_user) {
if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
hns_roce_db_unmap_user(
to_hr_ucontext(hr_qp->ibqp.uobject->context),
&hr_qp->rdb);
ib_umem_release(hr_qp->umem); ib_umem_release(hr_qp->umem);
} else { } else {
kfree(hr_qp->sq.wrid); kfree(hr_qp->sq.wrid);
......
...@@ -351,6 +351,11 @@ static struct ib_ucontext *hns_roce_alloc_ucontext(struct ib_device *ib_dev, ...@@ -351,6 +351,11 @@ static struct ib_ucontext *hns_roce_alloc_ucontext(struct ib_device *ib_dev,
if (ret) if (ret)
goto error_fail_uar_alloc; goto error_fail_uar_alloc;
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) {
INIT_LIST_HEAD(&context->page_list);
mutex_init(&context->page_mutex);
}
ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
if (ret) if (ret)
goto error_fail_copy_to_udata; goto error_fail_copy_to_udata;
......
...@@ -489,6 +489,15 @@ static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, ...@@ -489,6 +489,15 @@ static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev,
return 0; return 0;
} }
static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr)
{
if (attr->qp_type == IB_QPT_XRC_INI ||
attr->qp_type == IB_QPT_XRC_TGT || attr->srq)
return 0;
return 1;
}
static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
struct ib_pd *ib_pd, struct ib_pd *ib_pd,
struct ib_qp_init_attr *init_attr, struct ib_qp_init_attr *init_attr,
...@@ -497,6 +506,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ...@@ -497,6 +506,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
{ {
struct device *dev = hr_dev->dev; struct device *dev = hr_dev->dev;
struct hns_roce_ib_create_qp ucmd; struct hns_roce_ib_create_qp ucmd;
struct hns_roce_ib_create_qp_resp resp;
unsigned long qpn = 0; unsigned long qpn = 0;
int ret = 0; int ret = 0;
u32 page_shift; u32 page_shift;
...@@ -602,6 +612,18 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ...@@ -602,6 +612,18 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
dev_err(dev, "hns_roce_ib_umem_write_mtt error for create qp\n"); dev_err(dev, "hns_roce_ib_umem_write_mtt error for create qp\n");
goto err_mtt; goto err_mtt;
} }
if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
(udata->outlen == sizeof(resp)) &&
hns_roce_qp_has_rq(init_attr)) {
ret = hns_roce_db_map_user(
to_hr_ucontext(ib_pd->uobject->context),
ucmd.db_addr, &hr_qp->rdb);
if (ret) {
dev_err(dev, "rp record doorbell map failed!\n");
goto err_mtt;
}
}
} else { } else {
if (init_attr->create_flags & if (init_attr->create_flags &
IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) { IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
...@@ -698,17 +720,44 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ...@@ -698,17 +720,44 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
else else
hr_qp->doorbell_qpn = cpu_to_le64(hr_qp->qpn); hr_qp->doorbell_qpn = cpu_to_le64(hr_qp->qpn);
if (ib_pd->uobject && (udata->outlen == sizeof(resp)) &&
(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)) {
/* indicate kernel supports record db */
resp.cap_flags |= HNS_ROCE_SUPPORT_RQ_RECORD_DB;
ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
if (ret)
goto err_qp;
hr_qp->rdb_en = 1;
}
hr_qp->event = hns_roce_ib_qp_event; hr_qp->event = hns_roce_ib_qp_event;
return 0; return 0;
err_qp:
if (init_attr->qp_type == IB_QPT_GSI &&
hr_dev->hw_rev == HNS_ROCE_HW_VER1)
hns_roce_qp_remove(hr_dev, hr_qp);
else
hns_roce_qp_free(hr_dev, hr_qp);
err_qpn: err_qpn:
if (!sqpn) if (!sqpn)
hns_roce_release_range_qp(hr_dev, qpn, 1); hns_roce_release_range_qp(hr_dev, qpn, 1);
err_wrid: err_wrid:
kfree(hr_qp->sq.wrid); if (ib_pd->uobject) {
kfree(hr_qp->rq.wrid); if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
(udata->outlen == sizeof(resp)) &&
hns_roce_qp_has_rq(init_attr))
hns_roce_db_unmap_user(
to_hr_ucontext(ib_pd->uobject->context),
&hr_qp->rdb);
} else {
kfree(hr_qp->sq.wrid);
kfree(hr_qp->rq.wrid);
}
err_mtt: err_mtt:
hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
......
...@@ -49,7 +49,12 @@ struct hns_roce_ib_create_qp { ...@@ -49,7 +49,12 @@ struct hns_roce_ib_create_qp {
__u8 reserved[5]; __u8 reserved[5];
}; };
struct hns_roce_ib_create_qp_resp {
__u64 cap_flags;
};
struct hns_roce_ib_alloc_ucontext_resp { struct hns_roce_ib_alloc_ucontext_resp {
__u32 qp_tab_size; __u32 qp_tab_size;
__u32 reserved;
}; };
#endif /* HNS_ABI_USER_H */ #endif /* HNS_ABI_USER_H */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment