Commit 40589e74 authored by Andy Grover's avatar Andy Grover

RDS: Base init_depth and responder_resources on hw values

Instead of using a constant for initiator_depth and
responder_resources, read the per-QP values when the
device is enumerated, and then use these values when creating
the connection.
Signed-off-by: default avatarAndy Grover <andy.grover@oracle.com>
parent 15133f6e
...@@ -91,6 +91,9 @@ void rds_ib_add_one(struct ib_device *device) ...@@ -91,6 +91,9 @@ void rds_ib_add_one(struct ib_device *device)
min_t(unsigned int, dev_attr->max_fmr, fmr_pool_size) : min_t(unsigned int, dev_attr->max_fmr, fmr_pool_size) :
fmr_pool_size; fmr_pool_size;
rds_ibdev->max_initiator_depth = dev_attr->max_qp_init_rd_atom;
rds_ibdev->max_responder_resources = dev_attr->max_qp_rd_atom;
rds_ibdev->dev = device; rds_ibdev->dev = device;
rds_ibdev->pd = ib_alloc_pd(device); rds_ibdev->pd = ib_alloc_pd(device);
if (IS_ERR(rds_ibdev->pd)) if (IS_ERR(rds_ibdev->pd))
......
...@@ -164,6 +164,8 @@ struct rds_ib_device { ...@@ -164,6 +164,8 @@ struct rds_ib_device {
unsigned int max_fmrs; unsigned int max_fmrs;
int max_sge; int max_sge;
unsigned int max_wrs; unsigned int max_wrs;
unsigned int max_initiator_depth;
unsigned int max_responder_resources;
spinlock_t spinlock; /* protect the above */ spinlock_t spinlock; /* protect the above */
}; };
......
...@@ -153,18 +153,25 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even ...@@ -153,18 +153,25 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even
static void rds_ib_cm_fill_conn_param(struct rds_connection *conn, static void rds_ib_cm_fill_conn_param(struct rds_connection *conn,
struct rdma_conn_param *conn_param, struct rdma_conn_param *conn_param,
struct rds_ib_connect_private *dp, struct rds_ib_connect_private *dp,
u32 protocol_version) u32 protocol_version,
u32 max_responder_resources,
u32 max_initiator_depth)
{ {
struct rds_ib_connection *ic = conn->c_transport_data;
struct rds_ib_device *rds_ibdev;
memset(conn_param, 0, sizeof(struct rdma_conn_param)); memset(conn_param, 0, sizeof(struct rdma_conn_param));
/* XXX tune these? */
conn_param->responder_resources = 1; rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rds_ib_client);
conn_param->initiator_depth = 1;
conn_param->responder_resources =
min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources);
conn_param->initiator_depth =
min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth);
conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7); conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7);
conn_param->rnr_retry_count = 7; conn_param->rnr_retry_count = 7;
if (dp) { if (dp) {
struct rds_ib_connection *ic = conn->c_transport_data;
memset(dp, 0, sizeof(*dp)); memset(dp, 0, sizeof(*dp));
dp->dp_saddr = conn->c_laddr; dp->dp_saddr = conn->c_laddr;
dp->dp_daddr = conn->c_faddr; dp->dp_daddr = conn->c_faddr;
...@@ -479,7 +486,9 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, ...@@ -479,7 +486,9 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
goto out; goto out;
} }
rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version); rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version,
event->param.conn.responder_resources,
event->param.conn.initiator_depth);
/* rdma_accept() calls rdma_reject() internally if it fails */ /* rdma_accept() calls rdma_reject() internally if it fails */
err = rdma_accept(cm_id, &conn_param); err = rdma_accept(cm_id, &conn_param);
...@@ -516,8 +525,8 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id) ...@@ -516,8 +525,8 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id)
goto out; goto out;
} }
rds_ib_cm_fill_conn_param(conn, &conn_param, &dp, RDS_PROTOCOL_VERSION); rds_ib_cm_fill_conn_param(conn, &conn_param, &dp, RDS_PROTOCOL_VERSION,
UINT_MAX, UINT_MAX);
ret = rdma_connect(cm_id, &conn_param); ret = rdma_connect(cm_id, &conn_param);
if (ret) if (ret)
rds_ib_conn_error(conn, "rdma_connect failed (%d)\n", ret); rds_ib_conn_error(conn, "rdma_connect failed (%d)\n", ret);
......
...@@ -745,7 +745,6 @@ int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm, ...@@ -745,7 +745,6 @@ int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
rm->atomic.op_swap_add = args->fadd.add; rm->atomic.op_swap_add = args->fadd.add;
} }
rm->m_rdma_cookie = args->cookie;
rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME); rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);
rm->atomic.op_recverr = rs->rs_recverr; rm->atomic.op_recverr = rs->rs_recverr;
rm->atomic.op_sg = rds_message_alloc_sgs(rm, 1); rm->atomic.op_sg = rds_message_alloc_sgs(rm, 1);
...@@ -779,7 +778,7 @@ int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm, ...@@ -779,7 +778,7 @@ int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
rm->atomic.op_notifier->n_status = RDS_RDMA_SUCCESS; rm->atomic.op_notifier->n_status = RDS_RDMA_SUCCESS;
} }
rm->atomic.op_rkey = rds_rdma_cookie_key(rm->m_rdma_cookie); rm->atomic.op_rkey = rds_rdma_cookie_key(args->cookie);
rm->atomic.op_remote_addr = args->remote_addr + rds_rdma_cookie_offset(args->cookie); rm->atomic.op_remote_addr = args->remote_addr + rds_rdma_cookie_offset(args->cookie);
rm->atomic.op_active = 1; rm->atomic.op_active = 1;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment