Commit 49b53a93 authored by Steve Wise's avatar Steve Wise Committed by Doug Ledford

iw_cxgb4: add fast-path for small REG_MR operations

When processing a REG_MR work request, if fw supports the
FW_RI_NSMR_TPTE_WR work request, and if the page list for this
registration is <= 2 pages, and the current state of the mr is INVALID,
then use FW_RI_NSMR_TPTE_WR to pass down a fully populated TPTE for FW
to write.  This avoids FW having to do an async read of the TPTE blocking
the SQ until the read completes.

To know if the current MR state is INVALID or not, iw_cxgb4 must track the
state of each fastreg MR.  The c4iw_mr struct state is updated as REG_MR
and LOCAL_INV WRs are posted and completed, when a reg_mr is destroyed,
and when RECV completions are processed that include a local invalidation.

This optimization increases small IO IOPS for both iSER and NVMF.
Signed-off-by: default avatarSteve Wise <swise@opengridcomputing.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 086de575
...@@ -666,6 +666,18 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe, ...@@ -666,6 +666,18 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
return ret; return ret;
} }
static void invalidate_mr(struct c4iw_dev *rhp, u32 rkey)
{
struct c4iw_mr *mhp;
unsigned long flags;
spin_lock_irqsave(&rhp->lock, flags);
mhp = get_mhp(rhp, rkey >> 8);
if (mhp)
mhp->attr.state = 0;
spin_unlock_irqrestore(&rhp->lock, flags);
}
/* /*
* Get one cq entry from c4iw and map it to openib. * Get one cq entry from c4iw and map it to openib.
* *
...@@ -721,6 +733,7 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) ...@@ -721,6 +733,7 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_SE_INV) { CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_SE_INV) {
wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe); wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe);
wc->wc_flags |= IB_WC_WITH_INVALIDATE; wc->wc_flags |= IB_WC_WITH_INVALIDATE;
invalidate_mr(qhp->rhp, wc->ex.invalidate_rkey);
} }
} else { } else {
switch (CQE_OPCODE(&cqe)) { switch (CQE_OPCODE(&cqe)) {
...@@ -746,6 +759,10 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) ...@@ -746,6 +759,10 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
break; break;
case FW_RI_FAST_REGISTER: case FW_RI_FAST_REGISTER:
wc->opcode = IB_WC_REG_MR; wc->opcode = IB_WC_REG_MR;
/* Invalidate the MR if the fastreg failed */
if (CQE_STATUS(&cqe) != T4_ERR_SUCCESS)
invalidate_mr(qhp->rhp, CQE_WRID_FR_STAG(&cqe));
break; break;
default: default:
printk(KERN_ERR MOD "Unexpected opcode %d " printk(KERN_ERR MOD "Unexpected opcode %d "
......
...@@ -695,7 +695,7 @@ struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, ...@@ -695,7 +695,7 @@ struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
mhp->attr.pdid = php->pdid; mhp->attr.pdid = php->pdid;
mhp->attr.type = FW_RI_STAG_NSMR; mhp->attr.type = FW_RI_STAG_NSMR;
mhp->attr.stag = stag; mhp->attr.stag = stag;
mhp->attr.state = 1; mhp->attr.state = 0;
mmid = (stag) >> 8; mmid = (stag) >> 8;
mhp->ibmr.rkey = mhp->ibmr.lkey = stag; mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
......
...@@ -609,10 +609,42 @@ static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, ...@@ -609,10 +609,42 @@ static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe,
return 0; return 0;
} }
static void build_tpte_memreg(struct fw_ri_fr_nsmr_tpte_wr *fr,
struct ib_reg_wr *wr, struct c4iw_mr *mhp,
u8 *len16)
{
__be64 *p = (__be64 *)fr->pbl;
fr->r2 = cpu_to_be32(0);
fr->stag = cpu_to_be32(mhp->ibmr.rkey);
fr->tpte.valid_to_pdid = cpu_to_be32(FW_RI_TPTE_VALID_F |
FW_RI_TPTE_STAGKEY_V((mhp->ibmr.rkey & FW_RI_TPTE_STAGKEY_M)) |
FW_RI_TPTE_STAGSTATE_V(1) |
FW_RI_TPTE_STAGTYPE_V(FW_RI_STAG_NSMR) |
FW_RI_TPTE_PDID_V(mhp->attr.pdid));
fr->tpte.locread_to_qpid = cpu_to_be32(
FW_RI_TPTE_PERM_V(c4iw_ib_to_tpt_access(wr->access)) |
FW_RI_TPTE_ADDRTYPE_V(FW_RI_VA_BASED_TO) |
FW_RI_TPTE_PS_V(ilog2(wr->mr->page_size) - 12));
fr->tpte.nosnoop_pbladdr = cpu_to_be32(FW_RI_TPTE_PBLADDR_V(
PBL_OFF(&mhp->rhp->rdev, mhp->attr.pbl_addr)>>3));
fr->tpte.dca_mwbcnt_pstag = cpu_to_be32(0);
fr->tpte.len_hi = cpu_to_be32(0);
fr->tpte.len_lo = cpu_to_be32(mhp->ibmr.length);
fr->tpte.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32);
fr->tpte.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & 0xffffffff);
p[0] = cpu_to_be64((u64)mhp->mpl[0]);
p[1] = cpu_to_be64((u64)mhp->mpl[1]);
*len16 = DIV_ROUND_UP(sizeof(*fr), 16);
}
static int build_memreg(struct t4_sq *sq, union t4_wr *wqe, static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
struct ib_reg_wr *wr, u8 *len16, bool dsgl_supported) struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16,
bool dsgl_supported)
{ {
struct c4iw_mr *mhp = to_c4iw_mr(wr->mr);
struct fw_ri_immd *imdp; struct fw_ri_immd *imdp;
__be64 *p; __be64 *p;
int i; int i;
...@@ -674,9 +706,12 @@ static int build_memreg(struct t4_sq *sq, union t4_wr *wqe, ...@@ -674,9 +706,12 @@ static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
return 0; return 0;
} }
static int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr, static int build_inv_stag(struct c4iw_dev *dev, union t4_wr *wqe,
u8 *len16) struct ib_send_wr *wr, u8 *len16)
{ {
struct c4iw_mr *mhp = get_mhp(dev, wr->ex.invalidate_rkey >> 8);
mhp->attr.state = 0;
wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
wqe->inv.r2 = 0; wqe->inv.r2 = 0;
*len16 = DIV_ROUND_UP(sizeof wqe->inv, 16); *len16 = DIV_ROUND_UP(sizeof wqe->inv, 16);
...@@ -816,18 +851,32 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, ...@@ -816,18 +851,32 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
if (!qhp->wq.sq.oldest_read) if (!qhp->wq.sq.oldest_read)
qhp->wq.sq.oldest_read = swsqe; qhp->wq.sq.oldest_read = swsqe;
break; break;
case IB_WR_REG_MR: case IB_WR_REG_MR: {
fw_opcode = FW_RI_FR_NSMR_WR; struct c4iw_mr *mhp = to_c4iw_mr(reg_wr(wr)->mr);
swsqe->opcode = FW_RI_FAST_REGISTER; swsqe->opcode = FW_RI_FAST_REGISTER;
err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr), &len16, if (qhp->rhp->rdev.lldi.fr_nsmr_tpte_wr_support &&
!mhp->attr.state && mhp->mpl_len <= 2) {
fw_opcode = FW_RI_FR_NSMR_TPTE_WR;
build_tpte_memreg(&wqe->fr_tpte, reg_wr(wr),
mhp, &len16);
} else {
fw_opcode = FW_RI_FR_NSMR_WR;
err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr),
mhp, &len16,
qhp->rhp->rdev.lldi.ulptx_memwrite_dsgl); qhp->rhp->rdev.lldi.ulptx_memwrite_dsgl);
if (err)
break; break;
}
mhp->attr.state = 1;
break;
}
case IB_WR_LOCAL_INV: case IB_WR_LOCAL_INV:
if (wr->send_flags & IB_SEND_FENCE) if (wr->send_flags & IB_SEND_FENCE)
fw_flags |= FW_RI_LOCAL_FENCE_FLAG; fw_flags |= FW_RI_LOCAL_FENCE_FLAG;
fw_opcode = FW_RI_INV_LSTAG_WR; fw_opcode = FW_RI_INV_LSTAG_WR;
swsqe->opcode = FW_RI_LOCAL_INV; swsqe->opcode = FW_RI_LOCAL_INV;
err = build_inv_stag(wqe, wr, &len16); err = build_inv_stag(qhp->rhp, wqe, wr, &len16);
break; break;
default: default:
PDBG("%s post of type=%d TBD!\n", __func__, PDBG("%s post of type=%d TBD!\n", __func__,
......
...@@ -95,6 +95,7 @@ union t4_wr { ...@@ -95,6 +95,7 @@ union t4_wr {
struct fw_ri_rdma_read_wr read; struct fw_ri_rdma_read_wr read;
struct fw_ri_bind_mw_wr bind; struct fw_ri_bind_mw_wr bind;
struct fw_ri_fr_nsmr_wr fr; struct fw_ri_fr_nsmr_wr fr;
struct fw_ri_fr_nsmr_tpte_wr fr_tpte;
struct fw_ri_inv_lstag_wr inv; struct fw_ri_inv_lstag_wr inv;
struct t4_status_page status; struct t4_status_page status;
__be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_SQ_NUM_SLOTS]; __be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_SQ_NUM_SLOTS];
...@@ -170,7 +171,7 @@ struct t4_cqe { ...@@ -170,7 +171,7 @@ struct t4_cqe {
__be32 msn; __be32 msn;
} rcqe; } rcqe;
struct { struct {
u32 nada1; u32 stag;
u16 nada2; u16 nada2;
u16 cidx; u16 cidx;
} scqe; } scqe;
...@@ -232,6 +233,7 @@ struct t4_cqe { ...@@ -232,6 +233,7 @@ struct t4_cqe {
/* used for SQ completion processing */ /* used for SQ completion processing */
#define CQE_WRID_SQ_IDX(x) ((x)->u.scqe.cidx) #define CQE_WRID_SQ_IDX(x) ((x)->u.scqe.cidx)
#define CQE_WRID_FR_STAG(x) (be32_to_cpu((x)->u.scqe.stag))
/* generic accessor macros */ /* generic accessor macros */
#define CQE_WRID_HI(x) (be32_to_cpu((x)->u.gen.wrid_hi)) #define CQE_WRID_HI(x) (be32_to_cpu((x)->u.gen.wrid_hi))
......
...@@ -669,6 +669,18 @@ struct fw_ri_fr_nsmr_wr { ...@@ -669,6 +669,18 @@ struct fw_ri_fr_nsmr_wr {
#define FW_RI_FR_NSMR_WR_DCACPU_G(x) \ #define FW_RI_FR_NSMR_WR_DCACPU_G(x) \
(((x) >> FW_RI_FR_NSMR_WR_DCACPU_S) & FW_RI_FR_NSMR_WR_DCACPU_M) (((x) >> FW_RI_FR_NSMR_WR_DCACPU_S) & FW_RI_FR_NSMR_WR_DCACPU_M)
struct fw_ri_fr_nsmr_tpte_wr {
__u8 opcode;
__u8 flags;
__u16 wrid;
__u8 r1[3];
__u8 len16;
__u32 r2;
__u32 stag;
struct fw_ri_tpte tpte;
__u64 pbl[2];
};
struct fw_ri_inv_lstag_wr { struct fw_ri_inv_lstag_wr {
__u8 opcode; __u8 opcode;
__u8 flags; __u8 flags;
......
...@@ -100,6 +100,7 @@ enum fw_wr_opcodes { ...@@ -100,6 +100,7 @@ enum fw_wr_opcodes {
FW_RI_RECV_WR = 0x17, FW_RI_RECV_WR = 0x17,
FW_RI_BIND_MW_WR = 0x18, FW_RI_BIND_MW_WR = 0x18,
FW_RI_FR_NSMR_WR = 0x19, FW_RI_FR_NSMR_WR = 0x19,
FW_RI_FR_NSMR_TPTE_WR = 0x20,
FW_RI_INV_LSTAG_WR = 0x1a, FW_RI_INV_LSTAG_WR = 0x1a,
FW_ISCSI_TX_DATA_WR = 0x45, FW_ISCSI_TX_DATA_WR = 0x45,
FW_CRYPTO_LOOKASIDE_WR = 0X6d, FW_CRYPTO_LOOKASIDE_WR = 0X6d,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment