Commit d2832af3 authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker

xprtrdma: Clean up regbuf helpers

For code legibility, clean up the function names to be consistent
with the pattern: "rpcrdma" _ object-type _ action

Also rpcrdma_regbuf_alloc and rpcrdma_regbuf_free no longer have any
callers outside of verbs.c, and can thus be made static.
Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent 0f665ceb
...@@ -536,22 +536,21 @@ rpcrdma_unmap_sendctx(struct rpcrdma_sendctx *sc) ...@@ -536,22 +536,21 @@ rpcrdma_unmap_sendctx(struct rpcrdma_sendctx *sc)
/* Prepare an SGE for the RPC-over-RDMA transport header. /* Prepare an SGE for the RPC-over-RDMA transport header.
*/ */
static bool static bool rpcrdma_prepare_hdr_sge(struct rpcrdma_xprt *r_xprt,
rpcrdma_prepare_hdr_sge(struct rpcrdma_ia *ia, struct rpcrdma_req *req, struct rpcrdma_req *req, u32 len)
u32 len)
{ {
struct rpcrdma_sendctx *sc = req->rl_sendctx; struct rpcrdma_sendctx *sc = req->rl_sendctx;
struct rpcrdma_regbuf *rb = req->rl_rdmabuf; struct rpcrdma_regbuf *rb = req->rl_rdmabuf;
struct ib_sge *sge = sc->sc_sges; struct ib_sge *sge = sc->sc_sges;
if (!rpcrdma_dma_map_regbuf(ia, rb)) if (!rpcrdma_regbuf_dma_map(r_xprt, rb))
goto out_regbuf; goto out_regbuf;
sge->addr = rdmab_addr(rb); sge->addr = rdmab_addr(rb);
sge->length = len; sge->length = len;
sge->lkey = rdmab_lkey(rb); sge->lkey = rdmab_lkey(rb);
ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,
sge->length, DMA_TO_DEVICE); DMA_TO_DEVICE);
sc->sc_wr.num_sge++; sc->sc_wr.num_sge++;
return true; return true;
...@@ -563,22 +562,21 @@ rpcrdma_prepare_hdr_sge(struct rpcrdma_ia *ia, struct rpcrdma_req *req, ...@@ -563,22 +562,21 @@ rpcrdma_prepare_hdr_sge(struct rpcrdma_ia *ia, struct rpcrdma_req *req,
/* Prepare the Send SGEs. The head and tail iovec, and each entry /* Prepare the Send SGEs. The head and tail iovec, and each entry
* in the page list, gets its own SGE. * in the page list, gets its own SGE.
*/ */
static bool static bool rpcrdma_prepare_msg_sges(struct rpcrdma_xprt *r_xprt,
rpcrdma_prepare_msg_sges(struct rpcrdma_ia *ia, struct rpcrdma_req *req, struct rpcrdma_req *req,
struct xdr_buf *xdr, enum rpcrdma_chunktype rtype) struct xdr_buf *xdr,
enum rpcrdma_chunktype rtype)
{ {
struct rpcrdma_sendctx *sc = req->rl_sendctx; struct rpcrdma_sendctx *sc = req->rl_sendctx;
unsigned int sge_no, page_base, len, remaining; unsigned int sge_no, page_base, len, remaining;
struct rpcrdma_regbuf *rb = req->rl_sendbuf; struct rpcrdma_regbuf *rb = req->rl_sendbuf;
struct ib_device *device = ia->ri_device;
struct ib_sge *sge = sc->sc_sges; struct ib_sge *sge = sc->sc_sges;
u32 lkey = ia->ri_pd->local_dma_lkey;
struct page *page, **ppages; struct page *page, **ppages;
/* The head iovec is straightforward, as it is already /* The head iovec is straightforward, as it is already
* DMA-mapped. Sync the content that has changed. * DMA-mapped. Sync the content that has changed.
*/ */
if (!rpcrdma_dma_map_regbuf(ia, rb)) if (!rpcrdma_regbuf_dma_map(r_xprt, rb))
goto out_regbuf; goto out_regbuf;
sge_no = 1; sge_no = 1;
sge[sge_no].addr = rdmab_addr(rb); sge[sge_no].addr = rdmab_addr(rb);
...@@ -626,13 +624,14 @@ rpcrdma_prepare_msg_sges(struct rpcrdma_ia *ia, struct rpcrdma_req *req, ...@@ -626,13 +624,14 @@ rpcrdma_prepare_msg_sges(struct rpcrdma_ia *ia, struct rpcrdma_req *req,
goto out_mapping_overflow; goto out_mapping_overflow;
len = min_t(u32, PAGE_SIZE - page_base, remaining); len = min_t(u32, PAGE_SIZE - page_base, remaining);
sge[sge_no].addr = ib_dma_map_page(device, *ppages, sge[sge_no].addr =
page_base, len, ib_dma_map_page(rdmab_device(rb), *ppages,
DMA_TO_DEVICE); page_base, len, DMA_TO_DEVICE);
if (ib_dma_mapping_error(device, sge[sge_no].addr)) if (ib_dma_mapping_error(rdmab_device(rb),
sge[sge_no].addr))
goto out_mapping_err; goto out_mapping_err;
sge[sge_no].length = len; sge[sge_no].length = len;
sge[sge_no].lkey = lkey; sge[sge_no].lkey = rdmab_lkey(rb);
sc->sc_unmap_count++; sc->sc_unmap_count++;
ppages++; ppages++;
...@@ -653,13 +652,13 @@ rpcrdma_prepare_msg_sges(struct rpcrdma_ia *ia, struct rpcrdma_req *req, ...@@ -653,13 +652,13 @@ rpcrdma_prepare_msg_sges(struct rpcrdma_ia *ia, struct rpcrdma_req *req,
map_tail: map_tail:
sge_no++; sge_no++;
sge[sge_no].addr = ib_dma_map_page(device, page, sge[sge_no].addr =
page_base, len, ib_dma_map_page(rdmab_device(rb), page, page_base, len,
DMA_TO_DEVICE); DMA_TO_DEVICE);
if (ib_dma_mapping_error(device, sge[sge_no].addr)) if (ib_dma_mapping_error(rdmab_device(rb), sge[sge_no].addr))
goto out_mapping_err; goto out_mapping_err;
sge[sge_no].length = len; sge[sge_no].length = len;
sge[sge_no].lkey = lkey; sge[sge_no].lkey = rdmab_lkey(rb);
sc->sc_unmap_count++; sc->sc_unmap_count++;
} }
...@@ -707,11 +706,11 @@ rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, ...@@ -707,11 +706,11 @@ rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
req->rl_sendctx->sc_req = req; req->rl_sendctx->sc_req = req;
__clear_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags); __clear_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags);
if (!rpcrdma_prepare_hdr_sge(&r_xprt->rx_ia, req, hdrlen)) if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen))
return -EIO; return -EIO;
if (rtype != rpcrdma_areadch) if (rtype != rpcrdma_areadch)
if (!rpcrdma_prepare_msg_sges(&r_xprt->rx_ia, req, xdr, rtype)) if (!rpcrdma_prepare_msg_sges(r_xprt, req, xdr, rtype))
return -EIO; return -EIO;
return 0; return 0;
......
...@@ -76,7 +76,11 @@ ...@@ -76,7 +76,11 @@
static void rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc); static void rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc);
static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
static void rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf); static void rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf);
static void rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb); static struct rpcrdma_regbuf *
rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
gfp_t flags);
static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb);
static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
static void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp); static void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp);
/* Wait for outstanding transport work to finish. /* Wait for outstanding transport work to finish.
...@@ -437,11 +441,11 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia) ...@@ -437,11 +441,11 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia)
* mappings and MRs are gone. * mappings and MRs are gone.
*/ */
list_for_each_entry(rep, &buf->rb_recv_bufs, rr_list) list_for_each_entry(rep, &buf->rb_recv_bufs, rr_list)
rpcrdma_dma_unmap_regbuf(rep->rr_rdmabuf); rpcrdma_regbuf_dma_unmap(rep->rr_rdmabuf);
list_for_each_entry(req, &buf->rb_allreqs, rl_all) { list_for_each_entry(req, &buf->rb_allreqs, rl_all) {
rpcrdma_dma_unmap_regbuf(req->rl_rdmabuf); rpcrdma_regbuf_dma_unmap(req->rl_rdmabuf);
rpcrdma_dma_unmap_regbuf(req->rl_sendbuf); rpcrdma_regbuf_dma_unmap(req->rl_sendbuf);
rpcrdma_dma_unmap_regbuf(req->rl_recvbuf); rpcrdma_regbuf_dma_unmap(req->rl_recvbuf);
} }
rpcrdma_mrs_destroy(buf); rpcrdma_mrs_destroy(buf);
ib_dealloc_pd(ia->ri_pd); ib_dealloc_pd(ia->ri_pd);
...@@ -1014,17 +1018,17 @@ struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size, ...@@ -1014,17 +1018,17 @@ struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size,
if (req == NULL) if (req == NULL)
goto out1; goto out1;
rb = rpcrdma_alloc_regbuf(RPCRDMA_HDRBUF_SIZE, DMA_TO_DEVICE, flags); rb = rpcrdma_regbuf_alloc(RPCRDMA_HDRBUF_SIZE, DMA_TO_DEVICE, flags);
if (!rb) if (!rb)
goto out2; goto out2;
req->rl_rdmabuf = rb; req->rl_rdmabuf = rb;
xdr_buf_init(&req->rl_hdrbuf, rdmab_data(rb), rdmab_length(rb)); xdr_buf_init(&req->rl_hdrbuf, rdmab_data(rb), rdmab_length(rb));
req->rl_sendbuf = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, flags); req->rl_sendbuf = rpcrdma_regbuf_alloc(size, DMA_TO_DEVICE, flags);
if (!req->rl_sendbuf) if (!req->rl_sendbuf)
goto out3; goto out3;
req->rl_recvbuf = rpcrdma_alloc_regbuf(size, DMA_NONE, flags); req->rl_recvbuf = rpcrdma_regbuf_alloc(size, DMA_NONE, flags);
if (!req->rl_recvbuf) if (!req->rl_recvbuf)
goto out4; goto out4;
...@@ -1055,7 +1059,7 @@ static bool rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, bool temp) ...@@ -1055,7 +1059,7 @@ static bool rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, bool temp)
if (rep == NULL) if (rep == NULL)
goto out; goto out;
rep->rr_rdmabuf = rpcrdma_alloc_regbuf(cdata->inline_rsize, rep->rr_rdmabuf = rpcrdma_regbuf_alloc(cdata->inline_rsize,
DMA_FROM_DEVICE, GFP_KERNEL); DMA_FROM_DEVICE, GFP_KERNEL);
if (!rep->rr_rdmabuf) if (!rep->rr_rdmabuf)
goto out_free; goto out_free;
...@@ -1138,7 +1142,7 @@ rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) ...@@ -1138,7 +1142,7 @@ rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)
static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep) static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep)
{ {
rpcrdma_free_regbuf(rep->rr_rdmabuf); rpcrdma_regbuf_free(rep->rr_rdmabuf);
kfree(rep); kfree(rep);
} }
...@@ -1154,9 +1158,9 @@ rpcrdma_req_destroy(struct rpcrdma_req *req) ...@@ -1154,9 +1158,9 @@ rpcrdma_req_destroy(struct rpcrdma_req *req)
{ {
list_del(&req->rl_all); list_del(&req->rl_all);
rpcrdma_free_regbuf(req->rl_recvbuf); rpcrdma_regbuf_free(req->rl_recvbuf);
rpcrdma_free_regbuf(req->rl_sendbuf); rpcrdma_regbuf_free(req->rl_sendbuf);
rpcrdma_free_regbuf(req->rl_rdmabuf); rpcrdma_regbuf_free(req->rl_rdmabuf);
kfree(req); kfree(req);
} }
...@@ -1366,20 +1370,14 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) ...@@ -1366,20 +1370,14 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
} }
} }
/** /* Returns a pointer to a rpcrdma_regbuf object, or NULL.
* rpcrdma_alloc_regbuf - allocate and DMA-map memory for SEND/RECV buffers
* @size: size of buffer to be allocated, in bytes
* @direction: direction of data movement
* @flags: GFP flags
*
* Returns a pointer to a rpcrdma_regbuf object, or NULL.
* *
* xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for
* receiving the payload of RDMA RECV operations. During Long Calls * receiving the payload of RDMA RECV operations. During Long Calls
* or Replies they may be registered externally via frwr_map. * or Replies they may be registered externally via frwr_map.
*/ */
struct rpcrdma_regbuf * static struct rpcrdma_regbuf *
rpcrdma_alloc_regbuf(size_t size, enum dma_data_direction direction, rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
gfp_t flags) gfp_t flags)
{ {
struct rpcrdma_regbuf *rb; struct rpcrdma_regbuf *rb;
...@@ -1416,7 +1414,7 @@ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) ...@@ -1416,7 +1414,7 @@ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags)
if (!buf) if (!buf)
return false; return false;
rpcrdma_dma_unmap_regbuf(rb); rpcrdma_regbuf_dma_unmap(rb);
kfree(rb->rg_data); kfree(rb->rg_data);
rb->rg_data = buf; rb->rg_data = buf;
...@@ -1425,34 +1423,33 @@ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) ...@@ -1425,34 +1423,33 @@ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags)
} }
/** /**
* __rpcrdma_map_regbuf - DMA-map a regbuf * __rpcrdma_regbuf_dma_map - DMA-map a regbuf
* @ia: controlling rpcrdma_ia * @r_xprt: controlling transport instance
* @rb: regbuf to be mapped * @rb: regbuf to be mapped
*
* Returns true if the buffer is now DMA mapped to @r_xprt's device
*/ */
bool bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,
__rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb) struct rpcrdma_regbuf *rb)
{ {
struct ib_device *device = ia->ri_device; struct ib_device *device = r_xprt->rx_ia.ri_device;
if (rb->rg_direction == DMA_NONE) if (rb->rg_direction == DMA_NONE)
return false; return false;
rb->rg_iov.addr = ib_dma_map_single(device, rb->rg_iov.addr = ib_dma_map_single(device, rdmab_data(rb),
rdmab_data(rb), rdmab_length(rb), rb->rg_direction);
rdmab_length(rb),
rb->rg_direction);
if (ib_dma_mapping_error(device, rdmab_addr(rb))) { if (ib_dma_mapping_error(device, rdmab_addr(rb))) {
trace_xprtrdma_dma_maperr(rdmab_addr(rb)); trace_xprtrdma_dma_maperr(rdmab_addr(rb));
return false; return false;
} }
rb->rg_device = device; rb->rg_device = device;
rb->rg_iov.lkey = ia->ri_pd->local_dma_lkey; rb->rg_iov.lkey = r_xprt->rx_ia.ri_pd->local_dma_lkey;
return true; return true;
} }
static void static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb)
rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb)
{ {
if (!rb) if (!rb)
return; return;
...@@ -1460,19 +1457,14 @@ rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb) ...@@ -1460,19 +1457,14 @@ rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb)
if (!rpcrdma_regbuf_is_mapped(rb)) if (!rpcrdma_regbuf_is_mapped(rb))
return; return;
ib_dma_unmap_single(rb->rg_device, rdmab_addr(rb), ib_dma_unmap_single(rb->rg_device, rdmab_addr(rb), rdmab_length(rb),
rdmab_length(rb), rb->rg_direction); rb->rg_direction);
rb->rg_device = NULL; rb->rg_device = NULL;
} }
/** static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb)
* rpcrdma_free_regbuf - deregister and free registered buffer
* @rb: regbuf to be deregistered and freed
*/
void
rpcrdma_free_regbuf(struct rpcrdma_regbuf *rb)
{ {
rpcrdma_dma_unmap_regbuf(rb); rpcrdma_regbuf_dma_unmap(rb);
if (rb) if (rb)
kfree(rb->rg_data); kfree(rb->rg_data);
kfree(rb); kfree(rb);
...@@ -1547,12 +1539,10 @@ rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp) ...@@ -1547,12 +1539,10 @@ rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)
} }
rb = rep->rr_rdmabuf; rb = rep->rr_rdmabuf;
if (!rpcrdma_regbuf_is_mapped(rb)) { if (!rpcrdma_regbuf_dma_map(r_xprt, rb)) {
if (!__rpcrdma_dma_map_regbuf(&r_xprt->rx_ia, rb)) {
rpcrdma_recv_buffer_put(rep); rpcrdma_recv_buffer_put(rep);
break; break;
} }
}
trace_xprtrdma_post_recv(rep->rr_recv_wr.wr_cqe); trace_xprtrdma_post_recv(rep->rr_recv_wr.wr_cqe);
rep->rr_recv_wr.next = wr; rep->rr_recv_wr.next = wr;
......
...@@ -550,25 +550,34 @@ struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *); ...@@ -550,25 +550,34 @@ struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);
void rpcrdma_buffer_put(struct rpcrdma_req *); void rpcrdma_buffer_put(struct rpcrdma_req *);
void rpcrdma_recv_buffer_put(struct rpcrdma_rep *); void rpcrdma_recv_buffer_put(struct rpcrdma_rep *);
struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(size_t, enum dma_data_direction,
gfp_t);
bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size,
gfp_t flags); gfp_t flags);
bool __rpcrdma_dma_map_regbuf(struct rpcrdma_ia *, struct rpcrdma_regbuf *); bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,
void rpcrdma_free_regbuf(struct rpcrdma_regbuf *); struct rpcrdma_regbuf *rb);
static inline bool /**
rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf *rb) * rpcrdma_regbuf_is_mapped - check if buffer is DMA mapped
*
* Returns true if the buffer is now mapped to rb->rg_device.
*/
static inline bool rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf *rb)
{ {
return rb->rg_device != NULL; return rb->rg_device != NULL;
} }
static inline bool /**
rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb) * rpcrdma_regbuf_dma_map - DMA-map a regbuf
* @r_xprt: controlling transport instance
* @rb: regbuf to be mapped
*
* Returns true if the buffer is currently DMA mapped.
*/
static inline bool rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,
struct rpcrdma_regbuf *rb)
{ {
if (likely(rpcrdma_regbuf_is_mapped(rb))) if (likely(rpcrdma_regbuf_is_mapped(rb)))
return true; return true;
return __rpcrdma_dma_map_regbuf(ia, rb); return __rpcrdma_regbuf_dma_map(r_xprt, rb);
} }
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment