Commit 54cbd6b0 authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker

xprtrdma: Delay DMA mapping Send and Receive buffers

Currently, each regbuf is allocated and DMA mapped at the same time.
This is done during transport creation.

When a device driver is unloaded, every DMA-mapped buffer in use by
a transport has to be unmapped, and then remapped to the new
device if the driver is loaded again. Remapping will have to be done
_after_ the connect worker has set up the new device.

But there's an ordering problem:

call_allocate, which invokes xprt_rdma_allocate which calls
rpcrdma_alloc_regbuf to allocate Send buffers, happens _before_
the connect worker can run to set up the new device.

Instead, at transport creation, allocate each buffer, but leave it
unmapped. Once the RPC carries these buffers into ->send_request, by
which time a transport connection should have been established,
check to see that the RPC's buffers have been DMA mapped. If not,
map them there.

When device driver unplug support is added, it will simply unmap all
the transport's regbufs, but it doesn't have to deallocate the
underlying memory.
Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent 99ef4db3
...@@ -230,16 +230,24 @@ int rpcrdma_bc_marshal_reply(struct rpc_rqst *rqst) ...@@ -230,16 +230,24 @@ int rpcrdma_bc_marshal_reply(struct rpc_rqst *rqst)
__func__, (int)rpclen, rqst->rq_svec[0].iov_base); __func__, (int)rpclen, rqst->rq_svec[0].iov_base);
#endif #endif
if (!rpcrdma_dma_map_regbuf(&r_xprt->rx_ia, req->rl_rdmabuf))
goto out_map;
req->rl_send_iov[0].addr = rdmab_addr(req->rl_rdmabuf); req->rl_send_iov[0].addr = rdmab_addr(req->rl_rdmabuf);
req->rl_send_iov[0].length = RPCRDMA_HDRLEN_MIN; req->rl_send_iov[0].length = RPCRDMA_HDRLEN_MIN;
req->rl_send_iov[0].lkey = rdmab_lkey(req->rl_rdmabuf); req->rl_send_iov[0].lkey = rdmab_lkey(req->rl_rdmabuf);
if (!rpcrdma_dma_map_regbuf(&r_xprt->rx_ia, req->rl_sendbuf))
goto out_map;
req->rl_send_iov[1].addr = rdmab_addr(req->rl_sendbuf); req->rl_send_iov[1].addr = rdmab_addr(req->rl_sendbuf);
req->rl_send_iov[1].length = rpclen; req->rl_send_iov[1].length = rpclen;
req->rl_send_iov[1].lkey = rdmab_lkey(req->rl_sendbuf); req->rl_send_iov[1].lkey = rdmab_lkey(req->rl_sendbuf);
req->rl_niovs = 2; req->rl_niovs = 2;
return 0; return 0;
out_map:
pr_err("rpcrdma: failed to DMA map a Send buffer\n");
return -EIO;
} }
/** /**
......
...@@ -681,6 +681,8 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst) ...@@ -681,6 +681,8 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst)
transfertypes[rtype], transfertypes[wtype], transfertypes[rtype], transfertypes[wtype],
hdrlen, rpclen); hdrlen, rpclen);
if (!rpcrdma_dma_map_regbuf(&r_xprt->rx_ia, req->rl_rdmabuf))
goto out_map;
req->rl_send_iov[0].addr = rdmab_addr(req->rl_rdmabuf); req->rl_send_iov[0].addr = rdmab_addr(req->rl_rdmabuf);
req->rl_send_iov[0].length = hdrlen; req->rl_send_iov[0].length = hdrlen;
req->rl_send_iov[0].lkey = rdmab_lkey(req->rl_rdmabuf); req->rl_send_iov[0].lkey = rdmab_lkey(req->rl_rdmabuf);
...@@ -689,6 +691,8 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst) ...@@ -689,6 +691,8 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst)
if (rtype == rpcrdma_areadch) if (rtype == rpcrdma_areadch)
return 0; return 0;
if (!rpcrdma_dma_map_regbuf(&r_xprt->rx_ia, req->rl_sendbuf))
goto out_map;
req->rl_send_iov[1].addr = rdmab_addr(req->rl_sendbuf); req->rl_send_iov[1].addr = rdmab_addr(req->rl_sendbuf);
req->rl_send_iov[1].length = rpclen; req->rl_send_iov[1].length = rpclen;
req->rl_send_iov[1].lkey = rdmab_lkey(req->rl_sendbuf); req->rl_send_iov[1].lkey = rdmab_lkey(req->rl_sendbuf);
...@@ -704,6 +708,11 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst) ...@@ -704,6 +708,11 @@ rpcrdma_marshal_req(struct rpc_rqst *rqst)
out_unmap: out_unmap:
r_xprt->rx_ia.ri_ops->ro_unmap_safe(r_xprt, req, false); r_xprt->rx_ia.ri_ops->ro_unmap_safe(r_xprt, req, false);
return PTR_ERR(iptr); return PTR_ERR(iptr);
out_map:
pr_err("rpcrdma: failed to DMA map a Send buffer\n");
iptr = ERR_PTR(-EIO);
goto out_unmap;
} }
/* /*
......
...@@ -1179,9 +1179,8 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) ...@@ -1179,9 +1179,8 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
* @direction: direction of data movement * @direction: direction of data movement
* @flags: GFP flags * @flags: GFP flags
* *
* Returns an ERR_PTR, or a pointer to a regbuf, which is a * Returns an ERR_PTR, or a pointer to a regbuf, a buffer that
* contiguous memory region that is DMA mapped persistently, and * can be persistently DMA-mapped for I/O.
* is registered for local I/O.
* *
* xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for
* receiving the payload of RDMA RECV operations. During Long Calls * receiving the payload of RDMA RECV operations. During Long Calls
...@@ -1192,32 +1191,50 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, ...@@ -1192,32 +1191,50 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size,
enum dma_data_direction direction, gfp_t flags) enum dma_data_direction direction, gfp_t flags)
{ {
struct rpcrdma_regbuf *rb; struct rpcrdma_regbuf *rb;
struct ib_sge *iov;
rb = kmalloc(sizeof(*rb) + size, flags); rb = kmalloc(sizeof(*rb) + size, flags);
if (rb == NULL) if (rb == NULL)
goto out; return ERR_PTR(-ENOMEM);
rb->rg_device = NULL;
rb->rg_direction = direction; rb->rg_direction = direction;
iov = &rb->rg_iov; rb->rg_iov.length = size;
iov->length = size;
iov->lkey = ia->ri_pd->local_dma_lkey;
if (direction != DMA_NONE) {
iov->addr = ib_dma_map_single(ia->ri_device,
(void *)rb->rg_base,
rdmab_length(rb),
rb->rg_direction);
if (ib_dma_mapping_error(ia->ri_device, iov->addr))
goto out_free;
}
return rb; return rb;
}
out_free: /**
kfree(rb); * __rpcrdma_map_regbuf - DMA-map a regbuf
out: * @ia: controlling rpcrdma_ia
return ERR_PTR(-ENOMEM); * @rb: regbuf to be mapped
*/
bool
__rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
{
if (rb->rg_direction == DMA_NONE)
return false;
rb->rg_iov.addr = ib_dma_map_single(ia->ri_device,
(void *)rb->rg_base,
rdmab_length(rb),
rb->rg_direction);
if (ib_dma_mapping_error(ia->ri_device, rdmab_addr(rb)))
return false;
rb->rg_device = ia->ri_device;
rb->rg_iov.lkey = ia->ri_pd->local_dma_lkey;
return true;
}
static void
rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb)
{
if (!rpcrdma_regbuf_is_mapped(rb))
return;
ib_dma_unmap_single(rb->rg_device, rdmab_addr(rb),
rdmab_length(rb), rb->rg_direction);
rb->rg_device = NULL;
} }
/** /**
...@@ -1231,11 +1248,7 @@ rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb) ...@@ -1231,11 +1248,7 @@ rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
if (!rb) if (!rb)
return; return;
if (rb->rg_direction != DMA_NONE) { rpcrdma_dma_unmap_regbuf(rb);
ib_dma_unmap_single(ia->ri_device, rdmab_addr(rb),
rdmab_length(rb), rb->rg_direction);
}
kfree(rb); kfree(rb);
} }
...@@ -1307,11 +1320,17 @@ rpcrdma_ep_post_recv(struct rpcrdma_ia *ia, ...@@ -1307,11 +1320,17 @@ rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,
recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov; recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;
recv_wr.num_sge = 1; recv_wr.num_sge = 1;
if (!rpcrdma_dma_map_regbuf(ia, rep->rr_rdmabuf))
goto out_map;
rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail); rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);
if (rc) if (rc)
goto out_postrecv; goto out_postrecv;
return 0; return 0;
out_map:
pr_err("rpcrdma: failed to DMA map the Receive buffer\n");
return -EIO;
out_postrecv: out_postrecv:
pr_err("rpcrdma: ib_post_recv returned %i\n", rc); pr_err("rpcrdma: ib_post_recv returned %i\n", rc);
return -ENOTCONN; return -ENOTCONN;
......
...@@ -113,6 +113,7 @@ struct rpcrdma_ep { ...@@ -113,6 +113,7 @@ struct rpcrdma_ep {
struct rpcrdma_regbuf { struct rpcrdma_regbuf {
struct ib_sge rg_iov; struct ib_sge rg_iov;
struct ib_device *rg_device;
enum dma_data_direction rg_direction; enum dma_data_direction rg_direction;
__be32 rg_base[0] __attribute__ ((aligned(256))); __be32 rg_base[0] __attribute__ ((aligned(256)));
}; };
...@@ -480,9 +481,24 @@ void rpcrdma_defer_mr_recovery(struct rpcrdma_mw *); ...@@ -480,9 +481,24 @@ void rpcrdma_defer_mr_recovery(struct rpcrdma_mw *);
struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(struct rpcrdma_ia *, struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(struct rpcrdma_ia *,
size_t, enum dma_data_direction, size_t, enum dma_data_direction,
gfp_t); gfp_t);
bool __rpcrdma_dma_map_regbuf(struct rpcrdma_ia *, struct rpcrdma_regbuf *);
void rpcrdma_free_regbuf(struct rpcrdma_ia *, void rpcrdma_free_regbuf(struct rpcrdma_ia *,
struct rpcrdma_regbuf *); struct rpcrdma_regbuf *);
static inline bool
rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf *rb)
{
return rb->rg_device != NULL;
}
static inline bool
rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
{
if (likely(rpcrdma_regbuf_is_mapped(rb)))
return true;
return __rpcrdma_dma_map_regbuf(ia, rb);
}
int rpcrdma_ep_post_extra_recv(struct rpcrdma_xprt *, unsigned int); int rpcrdma_ep_post_extra_recv(struct rpcrdma_xprt *, unsigned int);
int rpcrdma_alloc_wq(void); int rpcrdma_alloc_wq(void);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment