Commit bfee5e32 authored by Vennila Megavannan's avatar Vennila Megavannan Committed by Doug Ledford

IB/rdmavt, staging/rdma/hfi1: use qps to dynamically scale timeout value

A busy_jiffies variable is maintained and updated when rc qps are
created and deleted. busy_jiffies is a scaled value of the number
of rc qps in the device. busy_jiffies is incremented every rc qp
scaling interval. busy_jiffies is added to the rc timeout
in add_retry_timer and mod_retry_timer. The rc qp scaling interval
is selected based on extensive performance evaluation of targeted
workloads.
Reviewed-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Reviewed-by: default avatarMike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: default avatarVennila Megavannan <vennila.megavannan@intel.com>
Signed-off-by: default avatarJubin John <jubin.john@intel.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 773d0451
...@@ -685,6 +685,19 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, ...@@ -685,6 +685,19 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
} }
rdi->n_qps_allocated++; rdi->n_qps_allocated++;
/*
* Maintain a busy_jiffies variable that will be added to the timeout
* period in mod_retry_timer and add_retry_timer. This busy jiffies
* is scaled by the number of rc qps created for the device to reduce
* the number of timeouts occurring when there is a large number of
* qps. busy_jiffies is incremented every rc qp scaling interval.
* The scaling interval is selected based on extensive performance
* evaluation of targeted workloads.
*/
if (init_attr->qp_type == IB_QPT_RC) {
rdi->n_rc_qps++;
rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL;
}
spin_unlock(&rdi->n_qps_lock); spin_unlock(&rdi->n_qps_lock);
if (qp->ip) { if (qp->ip) {
...@@ -1223,6 +1236,10 @@ int rvt_destroy_qp(struct ib_qp *ibqp) ...@@ -1223,6 +1236,10 @@ int rvt_destroy_qp(struct ib_qp *ibqp)
spin_lock(&rdi->n_qps_lock); spin_lock(&rdi->n_qps_lock);
rdi->n_qps_allocated--; rdi->n_qps_allocated--;
if (qp->ibqp.qp_type == IB_QPT_RC) {
rdi->n_rc_qps--;
rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL;
}
spin_unlock(&rdi->n_qps_lock); spin_unlock(&rdi->n_qps_lock);
if (qp->ip) if (qp->ip)
......
...@@ -68,9 +68,13 @@ ...@@ -68,9 +68,13 @@
*/ */
static inline void hfi1_add_retry_timer(struct rvt_qp *qp) static inline void hfi1_add_retry_timer(struct rvt_qp *qp)
{ {
struct ib_qp *ibqp = &qp->ibqp;
struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
qp->s_flags |= RVT_S_TIMER; qp->s_flags |= RVT_S_TIMER;
/* 4.096 usec. * (1 << qp->timeout) */ /* 4.096 usec. * (1 << qp->timeout) */
qp->s_timer.expires = jiffies + qp->timeout_jiffies; qp->s_timer.expires = jiffies + qp->timeout_jiffies +
rdi->busy_jiffies;
add_timer(&qp->s_timer); add_timer(&qp->s_timer);
} }
...@@ -99,9 +103,13 @@ void hfi1_add_rnr_timer(struct rvt_qp *qp, u32 to) ...@@ -99,9 +103,13 @@ void hfi1_add_rnr_timer(struct rvt_qp *qp, u32 to)
*/ */
static inline void hfi1_mod_retry_timer(struct rvt_qp *qp) static inline void hfi1_mod_retry_timer(struct rvt_qp *qp)
{ {
struct ib_qp *ibqp = &qp->ibqp;
struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
qp->s_flags |= RVT_S_TIMER; qp->s_flags |= RVT_S_TIMER;
/* 4.096 usec. * (1 << qp->timeout) */ /* 4.096 usec. * (1 << qp->timeout) */
mod_timer(&qp->s_timer, jiffies + qp->timeout_jiffies); mod_timer(&qp->s_timer, jiffies + qp->timeout_jiffies +
rdi->busy_jiffies);
} }
/** /**
......
...@@ -318,7 +318,9 @@ struct rvt_dev_info { ...@@ -318,7 +318,9 @@ struct rvt_dev_info {
/* QP */ /* QP */
struct rvt_qp_ibdev *qp_dev; struct rvt_qp_ibdev *qp_dev;
u32 n_qps_allocated; /* number of QPs allocated for device */ u32 n_qps_allocated; /* number of QPs allocated for device */
spinlock_t n_qps_lock; /* keep track of number of qps */ u32 n_rc_qps; /* number of RC QPs allocated for device */
u32 busy_jiffies; /* timeout scaling based on RC QP count */
spinlock_t n_qps_lock; /* protect qps, rc qps and busy jiffy counts */
/* memory maps */ /* memory maps */
struct list_head pending_mmaps; struct list_head pending_mmaps;
......
...@@ -225,6 +225,8 @@ struct rvt_ack_entry { ...@@ -225,6 +225,8 @@ struct rvt_ack_entry {
}; };
}; };
#define RC_QP_SCALING_INTERVAL 5
/* /*
* Variables prefixed with s_ are for the requester (sender). * Variables prefixed with s_ are for the requester (sender).
* Variables prefixed with r_ are for the responder (receiver). * Variables prefixed with r_ are for the responder (receiver).
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment