Commit 2acc5cae authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker

xprtrdma: Prevent dereferencing r_xprt->rx_ep after it is freed

r_xprt->rx_ep is known to be good while the transport's send lock is
held.  Otherwise additional references on rx_ep must be held when it
is used outside of that lock's critical sections.

For now, bump the rx_ep reference count once whenever there is at
least one outstanding Receive WR. This avoids the memory bandwidth
overhead of taking and releasing the reference count for every
ib_post_recv() and Receive completion.
Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent 48778464
...@@ -84,7 +84,8 @@ static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep); ...@@ -84,7 +84,8 @@ static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep);
static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt); static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);
static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
static int rpcrdma_ep_destroy(struct rpcrdma_ep *ep); static void rpcrdma_ep_get(struct rpcrdma_ep *ep);
static int rpcrdma_ep_put(struct rpcrdma_ep *ep);
static struct rpcrdma_regbuf * static struct rpcrdma_regbuf *
rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction, rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
gfp_t flags); gfp_t flags);
...@@ -97,7 +98,8 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb); ...@@ -97,7 +98,8 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
*/ */
static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt) static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)
{ {
struct rdma_cm_id *id = r_xprt->rx_ep->re_id; struct rpcrdma_ep *ep = r_xprt->rx_ep;
struct rdma_cm_id *id = ep->re_id;
/* Flush Receives, then wait for deferred Reply work /* Flush Receives, then wait for deferred Reply work
* to complete. * to complete.
...@@ -108,6 +110,8 @@ static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt) ...@@ -108,6 +110,8 @@ static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)
* local invalidations. * local invalidations.
*/ */
ib_drain_sq(id->qp); ib_drain_sq(id->qp);
rpcrdma_ep_put(ep);
} }
/** /**
...@@ -266,7 +270,7 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) ...@@ -266,7 +270,7 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
xprt_force_disconnect(xprt); xprt_force_disconnect(xprt);
goto disconnected; goto disconnected;
case RDMA_CM_EVENT_ESTABLISHED: case RDMA_CM_EVENT_ESTABLISHED:
kref_get(&ep->re_kref); rpcrdma_ep_get(ep);
ep->re_connect_status = 1; ep->re_connect_status = 1;
rpcrdma_update_cm_private(ep, &event->param.conn); rpcrdma_update_cm_private(ep, &event->param.conn);
trace_xprtrdma_inline_thresh(ep); trace_xprtrdma_inline_thresh(ep);
...@@ -289,7 +293,7 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) ...@@ -289,7 +293,7 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
ep->re_connect_status = -ECONNABORTED; ep->re_connect_status = -ECONNABORTED;
disconnected: disconnected:
xprt_force_disconnect(xprt); xprt_force_disconnect(xprt);
return rpcrdma_ep_destroy(ep); return rpcrdma_ep_put(ep);
default: default:
break; break;
} }
...@@ -345,7 +349,7 @@ static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt, ...@@ -345,7 +349,7 @@ static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt,
return ERR_PTR(rc); return ERR_PTR(rc);
} }
static void rpcrdma_ep_put(struct kref *kref) static void rpcrdma_ep_destroy(struct kref *kref)
{ {
struct rpcrdma_ep *ep = container_of(kref, struct rpcrdma_ep, re_kref); struct rpcrdma_ep *ep = container_of(kref, struct rpcrdma_ep, re_kref);
...@@ -369,13 +373,18 @@ static void rpcrdma_ep_put(struct kref *kref) ...@@ -369,13 +373,18 @@ static void rpcrdma_ep_put(struct kref *kref)
module_put(THIS_MODULE); module_put(THIS_MODULE);
} }
static noinline void rpcrdma_ep_get(struct rpcrdma_ep *ep)
{
kref_get(&ep->re_kref);
}
/* Returns: /* Returns:
* %0 if @ep still has a positive kref count, or * %0 if @ep still has a positive kref count, or
* %1 if @ep was destroyed successfully. * %1 if @ep was destroyed successfully.
*/ */
static int rpcrdma_ep_destroy(struct rpcrdma_ep *ep) static noinline int rpcrdma_ep_put(struct rpcrdma_ep *ep)
{ {
return kref_put(&ep->re_kref, rpcrdma_ep_put); return kref_put(&ep->re_kref, rpcrdma_ep_destroy);
} }
static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
...@@ -492,7 +501,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) ...@@ -492,7 +501,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
return 0; return 0;
out_destroy: out_destroy:
rpcrdma_ep_destroy(ep); rpcrdma_ep_put(ep);
rdma_destroy_id(id); rdma_destroy_id(id);
out_free: out_free:
kfree(ep); kfree(ep);
...@@ -521,8 +530,12 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) ...@@ -521,8 +530,12 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)
ep->re_connect_status = 0; ep->re_connect_status = 0;
xprt_clear_connected(xprt); xprt_clear_connected(xprt);
rpcrdma_reset_cwnd(r_xprt); rpcrdma_reset_cwnd(r_xprt);
/* Bump the ep's reference count while there are
* outstanding Receives.
*/
rpcrdma_ep_get(ep);
rpcrdma_post_recvs(r_xprt, true); rpcrdma_post_recvs(r_xprt, true);
rc = rpcrdma_sendctxs_create(r_xprt); rc = rpcrdma_sendctxs_create(r_xprt);
...@@ -587,7 +600,7 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) ...@@ -587,7 +600,7 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)
rpcrdma_mrs_destroy(r_xprt); rpcrdma_mrs_destroy(r_xprt);
rpcrdma_sendctxs_destroy(r_xprt); rpcrdma_sendctxs_destroy(r_xprt);
if (rpcrdma_ep_destroy(ep)) if (rpcrdma_ep_put(ep))
rdma_destroy_id(id); rdma_destroy_id(id);
r_xprt->rx_ep = NULL; r_xprt->rx_ep = NULL;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment