Commit d0f36c46 authored by Devesh Sharma's avatar Devesh Sharma Committed by Anna Schumaker

xprtrdma: take HCA driver refcount at client

This is a rework of the following patch sent almost a year back:
http://www.mail-archive.com/linux-rdma%40vger.kernel.org/msg20730.html

In presence of active mount if someone tries to rmmod vendor-driver, the
command remains stuck forever waiting for destruction of all rdma-cm-id.
in worst case client can crash during shutdown with active mounts.

The existing code assumes that ia->ri_id->device cannot change during
the lifetime of a transport. xprtrdma do not have support for
DEVICE_REMOVAL event either. Lifting that assumption and adding support
for DEVICE_REMOVAL event is a long chain of work, and is in plan.

The community decided that preventing the hang right now is more
important than waiting for architectural changes.

Thus, this patch introduces a temporary workaround to acquire HCA driver
module reference count during the mount of a nfs-rdma mount point.
Signed-off-by: default avatarDevesh Sharma <devesh.sharma@avagotech.com>
Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Reviewed-by: default avatarSagi Grimberg <sagig@dev.mellanox.co.il>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent 1241d7bf
...@@ -52,6 +52,7 @@ ...@@ -52,6 +52,7 @@
#include <linux/prefetch.h> #include <linux/prefetch.h>
#include <linux/sunrpc/addr.h> #include <linux/sunrpc/addr.h>
#include <asm/bitops.h> #include <asm/bitops.h>
#include <linux/module.h> /* try_module_get()/module_put() */
#include "xprt_rdma.h" #include "xprt_rdma.h"
...@@ -414,6 +415,14 @@ rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event) ...@@ -414,6 +415,14 @@ rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
return 0; return 0;
} }
static void rpcrdma_destroy_id(struct rdma_cm_id *id)
{
if (id) {
module_put(id->device->owner);
rdma_destroy_id(id);
}
}
static struct rdma_cm_id * static struct rdma_cm_id *
rpcrdma_create_id(struct rpcrdma_xprt *xprt, rpcrdma_create_id(struct rpcrdma_xprt *xprt,
struct rpcrdma_ia *ia, struct sockaddr *addr) struct rpcrdma_ia *ia, struct sockaddr *addr)
...@@ -440,6 +449,17 @@ rpcrdma_create_id(struct rpcrdma_xprt *xprt, ...@@ -440,6 +449,17 @@ rpcrdma_create_id(struct rpcrdma_xprt *xprt,
} }
wait_for_completion_interruptible_timeout(&ia->ri_done, wait_for_completion_interruptible_timeout(&ia->ri_done,
msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
/* FIXME:
* Until xprtrdma supports DEVICE_REMOVAL, the provider must
* be pinned while there are active NFS/RDMA mounts to prevent
* hangs and crashes at umount time.
*/
if (!ia->ri_async_rc && !try_module_get(id->device->owner)) {
dprintk("RPC: %s: Failed to get device module\n",
__func__);
ia->ri_async_rc = -ENODEV;
}
rc = ia->ri_async_rc; rc = ia->ri_async_rc;
if (rc) if (rc)
goto out; goto out;
...@@ -449,16 +469,17 @@ rpcrdma_create_id(struct rpcrdma_xprt *xprt, ...@@ -449,16 +469,17 @@ rpcrdma_create_id(struct rpcrdma_xprt *xprt,
if (rc) { if (rc) {
dprintk("RPC: %s: rdma_resolve_route() failed %i\n", dprintk("RPC: %s: rdma_resolve_route() failed %i\n",
__func__, rc); __func__, rc);
goto out; goto put;
} }
wait_for_completion_interruptible_timeout(&ia->ri_done, wait_for_completion_interruptible_timeout(&ia->ri_done,
msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
rc = ia->ri_async_rc; rc = ia->ri_async_rc;
if (rc) if (rc)
goto out; goto put;
return id; return id;
put:
module_put(id->device->owner);
out: out:
rdma_destroy_id(id); rdma_destroy_id(id);
return ERR_PTR(rc); return ERR_PTR(rc);
...@@ -566,7 +587,7 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg) ...@@ -566,7 +587,7 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
ib_dealloc_pd(ia->ri_pd); ib_dealloc_pd(ia->ri_pd);
ia->ri_pd = NULL; ia->ri_pd = NULL;
out2: out2:
rdma_destroy_id(ia->ri_id); rpcrdma_destroy_id(ia->ri_id);
ia->ri_id = NULL; ia->ri_id = NULL;
out1: out1:
return rc; return rc;
...@@ -584,7 +605,7 @@ rpcrdma_ia_close(struct rpcrdma_ia *ia) ...@@ -584,7 +605,7 @@ rpcrdma_ia_close(struct rpcrdma_ia *ia)
if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) { if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) {
if (ia->ri_id->qp) if (ia->ri_id->qp)
rdma_destroy_qp(ia->ri_id); rdma_destroy_qp(ia->ri_id);
rdma_destroy_id(ia->ri_id); rpcrdma_destroy_id(ia->ri_id);
ia->ri_id = NULL; ia->ri_id = NULL;
} }
...@@ -794,7 +815,7 @@ rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) ...@@ -794,7 +815,7 @@ rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
if (ia->ri_device != id->device) { if (ia->ri_device != id->device) {
printk("RPC: %s: can't reconnect on " printk("RPC: %s: can't reconnect on "
"different device!\n", __func__); "different device!\n", __func__);
rdma_destroy_id(id); rpcrdma_destroy_id(id);
rc = -ENETUNREACH; rc = -ENETUNREACH;
goto out; goto out;
} }
...@@ -803,7 +824,7 @@ rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) ...@@ -803,7 +824,7 @@ rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
if (rc) { if (rc) {
dprintk("RPC: %s: rdma_create_qp failed %i\n", dprintk("RPC: %s: rdma_create_qp failed %i\n",
__func__, rc); __func__, rc);
rdma_destroy_id(id); rpcrdma_destroy_id(id);
rc = -ENETUNREACH; rc = -ENETUNREACH;
goto out; goto out;
} }
...@@ -814,7 +835,7 @@ rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) ...@@ -814,7 +835,7 @@ rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
write_unlock(&ia->ri_qplock); write_unlock(&ia->ri_qplock);
rdma_destroy_qp(old); rdma_destroy_qp(old);
rdma_destroy_id(old); rpcrdma_destroy_id(old);
} else { } else {
dprintk("RPC: %s: connecting...\n", __func__); dprintk("RPC: %s: connecting...\n", __func__);
rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr); rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment