Commit fa658a98 authored by Steve Wise's avatar Steve Wise Committed by Roland Dreier

RDMA/cxgb4: Use the BAR2/WC path for kernel QPs and T5 devices

Signed-off-by: default avatarSteve Wise <swise@opengridcomputing.com>

[ Fix cast from u64* to integer.  - Roland ]
Signed-off-by: default avatarRoland Dreier <roland@purestorage.com>
parent 877f075a
...@@ -682,6 +682,9 @@ static void c4iw_dealloc(struct uld_ctx *ctx) ...@@ -682,6 +682,9 @@ static void c4iw_dealloc(struct uld_ctx *ctx)
idr_destroy(&ctx->dev->hwtid_idr); idr_destroy(&ctx->dev->hwtid_idr);
idr_destroy(&ctx->dev->stid_idr); idr_destroy(&ctx->dev->stid_idr);
idr_destroy(&ctx->dev->atid_idr); idr_destroy(&ctx->dev->atid_idr);
if (ctx->dev->rdev.bar2_kva)
iounmap(ctx->dev->rdev.bar2_kva);
if (ctx->dev->rdev.oc_mw_kva)
iounmap(ctx->dev->rdev.oc_mw_kva); iounmap(ctx->dev->rdev.oc_mw_kva);
ib_dealloc_device(&ctx->dev->ibdev); ib_dealloc_device(&ctx->dev->ibdev);
ctx->dev = NULL; ctx->dev = NULL;
...@@ -722,11 +725,31 @@ static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop) ...@@ -722,11 +725,31 @@ static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop)
} }
devp->rdev.lldi = *infop; devp->rdev.lldi = *infop;
devp->rdev.oc_mw_pa = pci_resource_start(devp->rdev.lldi.pdev, 2) + /*
(pci_resource_len(devp->rdev.lldi.pdev, 2) - * For T5 devices, we map all of BAR2 with WC.
roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size)); * For T4 devices with onchip qp mem, we map only that part
* of BAR2 with WC.
*/
devp->rdev.bar2_pa = pci_resource_start(devp->rdev.lldi.pdev, 2);
if (is_t5(devp->rdev.lldi.adapter_type)) {
devp->rdev.bar2_kva = ioremap_wc(devp->rdev.bar2_pa,
pci_resource_len(devp->rdev.lldi.pdev, 2));
if (!devp->rdev.bar2_kva) {
pr_err(MOD "Unable to ioremap BAR2\n");
return ERR_PTR(-EINVAL);
}
} else if (ocqp_supported(infop)) {
devp->rdev.oc_mw_pa =
pci_resource_start(devp->rdev.lldi.pdev, 2) +
pci_resource_len(devp->rdev.lldi.pdev, 2) -
roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size);
devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa, devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa,
devp->rdev.lldi.vr->ocq.size); devp->rdev.lldi.vr->ocq.size);
if (!devp->rdev.oc_mw_kva) {
pr_err(MOD "Unable to ioremap onchip mem\n");
return ERR_PTR(-EINVAL);
}
}
PDBG(KERN_INFO MOD "ocq memory: " PDBG(KERN_INFO MOD "ocq memory: "
"hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n", "hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n",
...@@ -1003,9 +1026,11 @@ static int enable_qp_db(int id, void *p, void *data) ...@@ -1003,9 +1026,11 @@ static int enable_qp_db(int id, void *p, void *data)
static void resume_rc_qp(struct c4iw_qp *qp) static void resume_rc_qp(struct c4iw_qp *qp)
{ {
spin_lock(&qp->lock); spin_lock(&qp->lock);
t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc); t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc,
is_t5(qp->rhp->rdev.lldi.adapter_type), NULL);
qp->wq.sq.wq_pidx_inc = 0; qp->wq.sq.wq_pidx_inc = 0;
t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc); t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc,
is_t5(qp->rhp->rdev.lldi.adapter_type), NULL);
qp->wq.rq.wq_pidx_inc = 0; qp->wq.rq.wq_pidx_inc = 0;
spin_unlock(&qp->lock); spin_unlock(&qp->lock);
} }
......
...@@ -149,6 +149,8 @@ struct c4iw_rdev { ...@@ -149,6 +149,8 @@ struct c4iw_rdev {
struct gen_pool *ocqp_pool; struct gen_pool *ocqp_pool;
u32 flags; u32 flags;
struct cxgb4_lld_info lldi; struct cxgb4_lld_info lldi;
unsigned long bar2_pa;
void __iomem *bar2_kva;
unsigned long oc_mw_pa; unsigned long oc_mw_pa;
void __iomem *oc_mw_kva; void __iomem *oc_mw_kva;
struct c4iw_stats stats; struct c4iw_stats stats;
......
...@@ -212,13 +212,23 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, ...@@ -212,13 +212,23 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
wq->db = rdev->lldi.db_reg; wq->db = rdev->lldi.db_reg;
wq->gts = rdev->lldi.gts_reg; wq->gts = rdev->lldi.gts_reg;
if (user || is_t5(rdev->lldi.adapter_type)) {
u32 off;
off = (wq->sq.qid << rdev->qpshift) & PAGE_MASK;
if (user) { if (user) {
wq->sq.udb = (u64)pci_resource_start(rdev->lldi.pdev, 2) + wq->sq.udb = (u64 __iomem *)(rdev->bar2_pa + off);
(wq->sq.qid << rdev->qpshift); } else {
wq->sq.udb &= PAGE_MASK; off += 128 * (wq->sq.qid & rdev->qpmask) + 8;
wq->rq.udb = (u64)pci_resource_start(rdev->lldi.pdev, 2) + wq->sq.udb = (u64 __iomem *)(rdev->bar2_kva + off);
(wq->rq.qid << rdev->qpshift); }
wq->rq.udb &= PAGE_MASK; off = (wq->rq.qid << rdev->qpshift) & PAGE_MASK;
if (user) {
wq->rq.udb = (u64 __iomem *)(rdev->bar2_pa + off);
} else {
off += 128 * (wq->rq.qid & rdev->qpmask) + 8;
wq->rq.udb = (u64 __iomem *)(rdev->bar2_kva + off);
}
} }
wq->rdev = rdev; wq->rdev = rdev;
wq->rq.msn = 1; wq->rq.msn = 1;
...@@ -299,9 +309,10 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, ...@@ -299,9 +309,10 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
if (ret) if (ret)
goto free_dma; goto free_dma;
PDBG("%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%llx rqudb 0x%llx\n", PDBG("%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%lx rqudb 0x%lx\n",
__func__, wq->sq.qid, wq->rq.qid, wq->db, __func__, wq->sq.qid, wq->rq.qid, wq->db,
(unsigned long long)wq->sq.udb, (unsigned long long)wq->rq.udb); (__force unsigned long) wq->sq.udb,
(__force unsigned long) wq->rq.udb);
return 0; return 0;
free_dma: free_dma:
...@@ -650,9 +661,10 @@ static int ring_kernel_sq_db(struct c4iw_qp *qhp, u16 inc) ...@@ -650,9 +661,10 @@ static int ring_kernel_sq_db(struct c4iw_qp *qhp, u16 inc)
spin_lock_irqsave(&qhp->rhp->lock, flags); spin_lock_irqsave(&qhp->rhp->lock, flags);
spin_lock(&qhp->lock); spin_lock(&qhp->lock);
if (qhp->rhp->db_state == NORMAL) { if (qhp->rhp->db_state == NORMAL)
t4_ring_sq_db(&qhp->wq, inc); t4_ring_sq_db(&qhp->wq, inc,
} else { is_t5(qhp->rhp->rdev.lldi.adapter_type), NULL);
else {
add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry); add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
qhp->wq.sq.wq_pidx_inc += inc; qhp->wq.sq.wq_pidx_inc += inc;
} }
...@@ -667,9 +679,10 @@ static int ring_kernel_rq_db(struct c4iw_qp *qhp, u16 inc) ...@@ -667,9 +679,10 @@ static int ring_kernel_rq_db(struct c4iw_qp *qhp, u16 inc)
spin_lock_irqsave(&qhp->rhp->lock, flags); spin_lock_irqsave(&qhp->rhp->lock, flags);
spin_lock(&qhp->lock); spin_lock(&qhp->lock);
if (qhp->rhp->db_state == NORMAL) { if (qhp->rhp->db_state == NORMAL)
t4_ring_rq_db(&qhp->wq, inc); t4_ring_rq_db(&qhp->wq, inc,
} else { is_t5(qhp->rhp->rdev.lldi.adapter_type), NULL);
else {
add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry); add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
qhp->wq.rq.wq_pidx_inc += inc; qhp->wq.rq.wq_pidx_inc += inc;
} }
...@@ -686,7 +699,7 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, ...@@ -686,7 +699,7 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
enum fw_wr_opcodes fw_opcode = 0; enum fw_wr_opcodes fw_opcode = 0;
enum fw_ri_wr_flags fw_flags; enum fw_ri_wr_flags fw_flags;
struct c4iw_qp *qhp; struct c4iw_qp *qhp;
union t4_wr *wqe; union t4_wr *wqe = NULL;
u32 num_wrs; u32 num_wrs;
struct t4_swsqe *swsqe; struct t4_swsqe *swsqe;
unsigned long flag; unsigned long flag;
...@@ -792,7 +805,8 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, ...@@ -792,7 +805,8 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
} }
if (!qhp->rhp->rdev.status_page->db_off) { if (!qhp->rhp->rdev.status_page->db_off) {
t4_ring_sq_db(&qhp->wq, idx); t4_ring_sq_db(&qhp->wq, idx,
is_t5(qhp->rhp->rdev.lldi.adapter_type), wqe);
spin_unlock_irqrestore(&qhp->lock, flag); spin_unlock_irqrestore(&qhp->lock, flag);
} else { } else {
spin_unlock_irqrestore(&qhp->lock, flag); spin_unlock_irqrestore(&qhp->lock, flag);
...@@ -806,7 +820,7 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, ...@@ -806,7 +820,7 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
{ {
int err = 0; int err = 0;
struct c4iw_qp *qhp; struct c4iw_qp *qhp;
union t4_recv_wr *wqe; union t4_recv_wr *wqe = NULL;
u32 num_wrs; u32 num_wrs;
u8 len16 = 0; u8 len16 = 0;
unsigned long flag; unsigned long flag;
...@@ -858,7 +872,8 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, ...@@ -858,7 +872,8 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
num_wrs--; num_wrs--;
} }
if (!qhp->rhp->rdev.status_page->db_off) { if (!qhp->rhp->rdev.status_page->db_off) {
t4_ring_rq_db(&qhp->wq, idx); t4_ring_rq_db(&qhp->wq, idx,
is_t5(qhp->rhp->rdev.lldi.adapter_type), wqe);
spin_unlock_irqrestore(&qhp->lock, flag); spin_unlock_irqrestore(&qhp->lock, flag);
} else { } else {
spin_unlock_irqrestore(&qhp->lock, flag); spin_unlock_irqrestore(&qhp->lock, flag);
...@@ -1677,11 +1692,11 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ...@@ -1677,11 +1692,11 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
mm2->len = PAGE_ALIGN(qhp->wq.rq.memsize); mm2->len = PAGE_ALIGN(qhp->wq.rq.memsize);
insert_mmap(ucontext, mm2); insert_mmap(ucontext, mm2);
mm3->key = uresp.sq_db_gts_key; mm3->key = uresp.sq_db_gts_key;
mm3->addr = qhp->wq.sq.udb; mm3->addr = (__force unsigned long) qhp->wq.sq.udb;
mm3->len = PAGE_SIZE; mm3->len = PAGE_SIZE;
insert_mmap(ucontext, mm3); insert_mmap(ucontext, mm3);
mm4->key = uresp.rq_db_gts_key; mm4->key = uresp.rq_db_gts_key;
mm4->addr = qhp->wq.rq.udb; mm4->addr = (__force unsigned long) qhp->wq.rq.udb;
mm4->len = PAGE_SIZE; mm4->len = PAGE_SIZE;
insert_mmap(ucontext, mm4); insert_mmap(ucontext, mm4);
if (mm5) { if (mm5) {
......
...@@ -292,7 +292,7 @@ struct t4_sq { ...@@ -292,7 +292,7 @@ struct t4_sq {
unsigned long phys_addr; unsigned long phys_addr;
struct t4_swsqe *sw_sq; struct t4_swsqe *sw_sq;
struct t4_swsqe *oldest_read; struct t4_swsqe *oldest_read;
u64 udb; u64 __iomem *udb;
size_t memsize; size_t memsize;
u32 qid; u32 qid;
u16 in_use; u16 in_use;
...@@ -314,7 +314,7 @@ struct t4_rq { ...@@ -314,7 +314,7 @@ struct t4_rq {
dma_addr_t dma_addr; dma_addr_t dma_addr;
DEFINE_DMA_UNMAP_ADDR(mapping); DEFINE_DMA_UNMAP_ADDR(mapping);
struct t4_swrqe *sw_rq; struct t4_swrqe *sw_rq;
u64 udb; u64 __iomem *udb;
size_t memsize; size_t memsize;
u32 qid; u32 qid;
u32 msn; u32 msn;
...@@ -435,15 +435,67 @@ static inline u16 t4_sq_wq_size(struct t4_wq *wq) ...@@ -435,15 +435,67 @@ static inline u16 t4_sq_wq_size(struct t4_wq *wq)
return wq->sq.size * T4_SQ_NUM_SLOTS; return wq->sq.size * T4_SQ_NUM_SLOTS;
} }
static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc) /* This function copies 64 byte coalesced work request to memory
* mapped BAR2 space. For coalesced WRs, the SGE fetches data
* from the FIFO instead of from Host.
*/
static inline void pio_copy(u64 __iomem *dst, u64 *src)
{
int count = 8;
while (count) {
writeq(*src, dst);
src++;
dst++;
count--;
}
}
static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, u8 t5,
union t4_wr *wqe)
{ {
/* Flush host queue memory writes. */
wmb();
if (t5) {
if (inc == 1 && wqe) {
PDBG("%s: WC wq->sq.pidx = %d\n",
__func__, wq->sq.pidx);
pio_copy(wq->sq.udb + 7, (void *)wqe);
} else {
PDBG("%s: DB wq->sq.pidx = %d\n",
__func__, wq->sq.pidx);
writel(PIDX_T5(inc), wq->sq.udb);
}
/* Flush user doorbell area writes. */
wmb(); wmb();
return;
}
writel(QID(wq->sq.qid) | PIDX(inc), wq->db); writel(QID(wq->sq.qid) | PIDX(inc), wq->db);
} }
static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc) static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc, u8 t5,
union t4_recv_wr *wqe)
{ {
/* Flush host queue memory writes. */
wmb(); wmb();
if (t5) {
if (inc == 1 && wqe) {
PDBG("%s: WC wq->rq.pidx = %d\n",
__func__, wq->rq.pidx);
pio_copy(wq->rq.udb + 7, (void *)wqe);
} else {
PDBG("%s: DB wq->rq.pidx = %d\n",
__func__, wq->rq.pidx);
writel(PIDX_T5(inc), wq->rq.udb);
}
/* Flush user doorbell area writes. */
wmb();
return;
}
writel(QID(wq->rq.qid) | PIDX(inc), wq->db); writel(QID(wq->rq.qid) | PIDX(inc), wq->db);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment