Commit 1198fcea authored by Brian Welty's avatar Brian Welty Committed by Doug Ledford

IB/hfi1, rdmavt: Move SGE state helper routines into rdmavt

To improve code reuse, add small SGE state helper routines to rdmavt_mr.h.
Leverage these in hfi1, including refactoring of hfi1_copy_sge.
Reviewed-by: default avatarMike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: default avatarBrian Welty <brian.welty@intel.com>
Signed-off-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 0128fcea
...@@ -1797,8 +1797,6 @@ int kdeth_process_expected(struct hfi1_packet *packet); ...@@ -1797,8 +1797,6 @@ int kdeth_process_expected(struct hfi1_packet *packet);
int kdeth_process_eager(struct hfi1_packet *packet); int kdeth_process_eager(struct hfi1_packet *packet);
int process_receive_invalid(struct hfi1_packet *packet); int process_receive_invalid(struct hfi1_packet *packet);
void update_sge(struct rvt_sge_state *ss, u32 length);
/* global module parameter variables */ /* global module parameter variables */
extern unsigned int hfi1_max_mtu; extern unsigned int hfi1_max_mtu;
extern unsigned int hfi1_cu; extern unsigned int hfi1_cu;
......
...@@ -67,7 +67,7 @@ static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, ...@@ -67,7 +67,7 @@ static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe,
ss->sg_list = wqe->sg_list + 1; ss->sg_list = wqe->sg_list + 1;
ss->num_sge = wqe->wr.num_sge; ss->num_sge = wqe->wr.num_sge;
ss->total_len = wqe->length; ss->total_len = wqe->length;
hfi1_skip_sge(ss, len, false); rvt_skip_sge(ss, len, false);
return wqe->length - len; return wqe->length - len;
} }
......
...@@ -192,7 +192,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) ...@@ -192,7 +192,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
sizeof(grh), true, false); sizeof(grh), true, false);
wc.wc_flags |= IB_WC_GRH; wc.wc_flags |= IB_WC_GRH;
} else { } else {
hfi1_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true);
} }
ssge.sg_list = swqe->sg_list + 1; ssge.sg_list = swqe->sg_list + 1;
ssge.sge = *swqe->sg_list; ssge.sge = *swqe->sg_list;
...@@ -815,7 +815,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) ...@@ -815,7 +815,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
sizeof(struct ib_grh), true, false); sizeof(struct ib_grh), true, false);
wc.wc_flags |= IB_WC_GRH; wc.wc_flags |= IB_WC_GRH;
} else { } else {
hfi1_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true);
} }
hfi1_copy_sge(&qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), hfi1_copy_sge(&qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh),
true, false); true, false);
......
...@@ -462,12 +462,8 @@ void hfi1_copy_sge( ...@@ -462,12 +462,8 @@ void hfi1_copy_sge(
again: again:
while (length) { while (length) {
u32 len = sge->length; u32 len = rvt_get_sge_length(sge, length);
if (len > length)
len = length;
if (len > sge->sge_length)
len = sge->sge_length;
WARN_ON_ONCE(len == 0); WARN_ON_ONCE(len == 0);
if (unlikely(in_last)) { if (unlikely(in_last)) {
/* enforce byte transfer ordering */ /* enforce byte transfer ordering */
...@@ -478,25 +474,7 @@ void hfi1_copy_sge( ...@@ -478,25 +474,7 @@ void hfi1_copy_sge(
} else { } else {
memcpy(sge->vaddr, data, len); memcpy(sge->vaddr, data, len);
} }
sge->vaddr += len; rvt_update_sge(ss, len, release);
sge->length -= len;
sge->sge_length -= len;
if (sge->sge_length == 0) {
if (release)
rvt_put_mr(sge->mr);
if (--ss->num_sge)
*sge = *ss->sg_list++;
} else if (sge->length == 0 && sge->mr->lkey) {
if (++sge->n >= RVT_SEGSZ) {
if (++sge->m >= sge->mr->mapsz)
break;
sge->n = 0;
}
sge->vaddr =
sge->mr->map[sge->m]->segs[sge->n].vaddr;
sge->length =
sge->mr->map[sge->m]->segs[sge->n].length;
}
data += len; data += len;
length -= len; length -= len;
} }
...@@ -509,46 +487,6 @@ void hfi1_copy_sge( ...@@ -509,46 +487,6 @@ void hfi1_copy_sge(
} }
} }
/**
* hfi1_skip_sge - skip over SGE memory
* @ss: the SGE state
* @length: the number of bytes to skip
*/
void hfi1_skip_sge(struct rvt_sge_state *ss, u32 length, bool release)
{
struct rvt_sge *sge = &ss->sge;
while (length) {
u32 len = sge->length;
if (len > length)
len = length;
if (len > sge->sge_length)
len = sge->sge_length;
WARN_ON_ONCE(len == 0);
sge->vaddr += len;
sge->length -= len;
sge->sge_length -= len;
if (sge->sge_length == 0) {
if (release)
rvt_put_mr(sge->mr);
if (--ss->num_sge)
*sge = *ss->sg_list++;
} else if (sge->length == 0 && sge->mr->lkey) {
if (++sge->n >= RVT_SEGSZ) {
if (++sge->m >= sge->mr->mapsz)
break;
sge->n = 0;
}
sge->vaddr =
sge->mr->map[sge->m]->segs[sge->n].vaddr;
sge->length =
sge->mr->map[sge->m]->segs[sge->n].length;
}
length -= len;
}
}
/* /*
* Make sure the QP is ready and able to accept the given opcode. * Make sure the QP is ready and able to accept the given opcode.
*/ */
...@@ -690,27 +628,6 @@ static void mem_timer(unsigned long data) ...@@ -690,27 +628,6 @@ static void mem_timer(unsigned long data)
hfi1_qp_wakeup(qp, RVT_S_WAIT_KMEM); hfi1_qp_wakeup(qp, RVT_S_WAIT_KMEM);
} }
void update_sge(struct rvt_sge_state *ss, u32 length)
{
struct rvt_sge *sge = &ss->sge;
sge->vaddr += length;
sge->length -= length;
sge->sge_length -= length;
if (sge->sge_length == 0) {
if (--ss->num_sge)
*sge = *ss->sg_list++;
} else if (sge->length == 0 && sge->mr->lkey) {
if (++sge->n >= RVT_SEGSZ) {
if (++sge->m >= sge->mr->mapsz)
return;
sge->n = 0;
}
sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
}
}
/* /*
* This is called with progress side lock held. * This is called with progress side lock held.
*/ */
...@@ -799,7 +716,7 @@ static noinline int build_verbs_ulp_payload( ...@@ -799,7 +716,7 @@ static noinline int build_verbs_ulp_payload(
len); len);
if (ret) if (ret)
goto bail_txadd; goto bail_txadd;
update_sge(ss, len); rvt_update_sge(ss, len, false);
length -= len; length -= len;
} }
return ret; return ret;
...@@ -1074,7 +991,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, ...@@ -1074,7 +991,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
if (slen > len) if (slen > len)
slen = len; slen = len;
update_sge(ss, slen); rvt_update_sge(ss, slen, false);
seg_pio_copy_mid(pbuf, addr, slen); seg_pio_copy_mid(pbuf, addr, slen);
len -= slen; len -= slen;
} }
......
...@@ -291,8 +291,6 @@ int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps); ...@@ -291,8 +291,6 @@ int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
void hfi1_copy_sge(struct rvt_sge_state *ss, void *data, u32 length, void hfi1_copy_sge(struct rvt_sge_state *ss, void *data, u32 length,
bool release, bool copy_last); bool release, bool copy_last);
void hfi1_skip_sge(struct rvt_sge_state *ss, u32 length, bool release);
void hfi1_cnp_rcv(struct hfi1_packet *packet); void hfi1_cnp_rcv(struct hfi1_packet *packet);
void hfi1_uc_rcv(struct hfi1_packet *packet); void hfi1_uc_rcv(struct hfi1_packet *packet);
......
...@@ -141,4 +141,54 @@ static inline void rvt_put_ss(struct rvt_sge_state *ss) ...@@ -141,4 +141,54 @@ static inline void rvt_put_ss(struct rvt_sge_state *ss)
} }
} }
static inline u32 rvt_get_sge_length(struct rvt_sge *sge, u32 length)
{
u32 len = sge->length;
if (len > length)
len = length;
if (len > sge->sge_length)
len = sge->sge_length;
return len;
}
static inline void rvt_update_sge(struct rvt_sge_state *ss, u32 length,
bool release)
{
struct rvt_sge *sge = &ss->sge;
sge->vaddr += length;
sge->length -= length;
sge->sge_length -= length;
if (sge->sge_length == 0) {
if (release)
rvt_put_mr(sge->mr);
if (--ss->num_sge)
*sge = *ss->sg_list++;
} else if (sge->length == 0 && sge->mr->lkey) {
if (++sge->n >= RVT_SEGSZ) {
if (++sge->m >= sge->mr->mapsz)
return;
sge->n = 0;
}
sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
}
}
static inline void rvt_skip_sge(struct rvt_sge_state *ss, u32 length,
bool release)
{
struct rvt_sge *sge = &ss->sge;
while (length) {
u32 len = rvt_get_sge_length(sge, length);
WARN_ON_ONCE(len == 0);
rvt_update_sge(ss, len, release);
length -= len;
}
}
#endif /* DEF_RDMAVT_INCMRH */ #endif /* DEF_RDMAVT_INCMRH */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment