Commit 09b93088 authored by Or Gerlitz's avatar Or Gerlitz Committed by Roland Dreier

IB: Add a QP creation flag to use GFP_NOIO allocations

This addresses a problem where NFS client writes over IPoIB connected
mode may deadlock on memory allocation/writeback.

The problem is not directly memory reclamation.  There is an indirect
dependency between network filesystems writing back pages and
ipoib_cm_tx_init() due to how a kworker is used.  Page reclaim cannot
make forward progress until ipoib_cm_tx_init() succeeds and it is
stuck in page reclaim itself waiting for network transmission.
Ordinarily this situation may be avoided by having the caller use
GFP_NOFS but ipoib_cm_tx_init() does not have that information.

To address this, take a general approach and add a new QP creation
flag that tells the low-level hardware driver to use GFP_NOIO for the
memory allocations related to the new QP.

Use the new flag in the ipoib connected mode path, and if the driver
doesn't support it, re-issue the QP creation without the flag.
Signed-off-by: default avatarMel Gorman <mgorman@suse.de>
Signed-off-by: default avatarJiri Kosina <jkosina@suse.cz>
Signed-off-by: default avatarOr Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: default avatarRoland Dreier <roland@purestorage.com>
parent 60093dc0
...@@ -1030,10 +1030,20 @@ static struct ib_qp *ipoib_cm_create_tx_qp(struct net_device *dev, struct ipoib_ ...@@ -1030,10 +1030,20 @@ static struct ib_qp *ipoib_cm_create_tx_qp(struct net_device *dev, struct ipoib_
.cap.max_send_sge = 1, .cap.max_send_sge = 1,
.sq_sig_type = IB_SIGNAL_ALL_WR, .sq_sig_type = IB_SIGNAL_ALL_WR,
.qp_type = IB_QPT_RC, .qp_type = IB_QPT_RC,
.qp_context = tx .qp_context = tx,
.create_flags = IB_QP_CREATE_USE_GFP_NOIO
}; };
return ib_create_qp(priv->pd, &attr); struct ib_qp *tx_qp;
tx_qp = ib_create_qp(priv->pd, &attr);
if (PTR_ERR(tx_qp) == -EINVAL) {
ipoib_warn(priv, "can't use GFP_NOIO for QPs on device %s, using GFP_KERNEL\n",
priv->ca->name);
attr.create_flags &= ~IB_QP_CREATE_USE_GFP_NOIO;
tx_qp = ib_create_qp(priv->pd, &attr);
}
return tx_qp;
} }
static int ipoib_cm_send_req(struct net_device *dev, static int ipoib_cm_send_req(struct net_device *dev,
...@@ -1104,12 +1114,14 @@ static int ipoib_cm_tx_init(struct ipoib_cm_tx *p, u32 qpn, ...@@ -1104,12 +1114,14 @@ static int ipoib_cm_tx_init(struct ipoib_cm_tx *p, u32 qpn,
struct ipoib_dev_priv *priv = netdev_priv(p->dev); struct ipoib_dev_priv *priv = netdev_priv(p->dev);
int ret; int ret;
p->tx_ring = vzalloc(ipoib_sendq_size * sizeof *p->tx_ring); p->tx_ring = __vmalloc(ipoib_sendq_size * sizeof *p->tx_ring,
GFP_NOIO, PAGE_KERNEL);
if (!p->tx_ring) { if (!p->tx_ring) {
ipoib_warn(priv, "failed to allocate tx ring\n"); ipoib_warn(priv, "failed to allocate tx ring\n");
ret = -ENOMEM; ret = -ENOMEM;
goto err_tx; goto err_tx;
} }
memset(p->tx_ring, 0, ipoib_sendq_size * sizeof *p->tx_ring);
p->qp = ipoib_cm_create_tx_qp(p->dev, p); p->qp = ipoib_cm_create_tx_qp(p->dev, p);
if (IS_ERR(p->qp)) { if (IS_ERR(p->qp)) {
......
...@@ -783,6 +783,7 @@ enum ib_qp_create_flags { ...@@ -783,6 +783,7 @@ enum ib_qp_create_flags {
IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK = 1 << 1, IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK = 1 << 1,
IB_QP_CREATE_NETIF_QP = 1 << 5, IB_QP_CREATE_NETIF_QP = 1 << 5,
IB_QP_CREATE_SIGNATURE_EN = 1 << 6, IB_QP_CREATE_SIGNATURE_EN = 1 << 6,
IB_QP_CREATE_USE_GFP_NOIO = 1 << 7,
/* reserve bits 26-31 for low level drivers' internal use */ /* reserve bits 26-31 for low level drivers' internal use */
IB_QP_CREATE_RESERVED_START = 1 << 26, IB_QP_CREATE_RESERVED_START = 1 << 26,
IB_QP_CREATE_RESERVED_END = 1 << 31, IB_QP_CREATE_RESERVED_END = 1 << 31,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment