Commit 4b5e5f41 authored by Bart Van Assche's avatar Bart Van Assche Committed by Roland Dreier

IB/srp: Make HCA completion vector configurable

Several InfiniBand HCAs allow configuring the completion vector per
CQ.  This allows spreading the workload created by IB completion
interrupts over multiple MSI-X vectors and hence over multiple CPU
cores.  In other words, configuring the completion vector properly not
only allows reducing latency on an initiator connected to multiple
SRP targets but also allows improving throughput.
Signed-off-by: default avatarBart Van Assche <bvanassche@acm.org>
Acked-by: default avatarDavid Dillow <dillowda@ornl.gov>
Signed-off-by: default avatarRoland Dreier <roland@purestorage.com>
parent 96fc248a
...@@ -54,6 +54,13 @@ Description: Interface for making ib_srp connect to a new target. ...@@ -54,6 +54,13 @@ Description: Interface for making ib_srp connect to a new target.
ib_srp. Specifying a value that exceeds cmd_sg_entries is ib_srp. Specifying a value that exceeds cmd_sg_entries is
only safe with partial memory descriptor list support enabled only safe with partial memory descriptor list support enabled
(allow_ext_sg=1). (allow_ext_sg=1).
* comp_vector, a number in the range 0..n-1 specifying the
MSI-X completion vector. Some HCA's allocate multiple (n)
MSI-X vectors per HCA port. If the IRQ affinity masks of
these interrupts have been configured such that each MSI-X
interrupt is handled by a different CPU then the comp_vector
parameter can be used to spread the SRP completion workload
over multiple CPU's.
What: /sys/class/infiniband_srp/srp-<hca>-<port_number>/ibdev What: /sys/class/infiniband_srp/srp-<hca>-<port_number>/ibdev
Date: January 2, 2006 Date: January 2, 2006
......
...@@ -231,14 +231,16 @@ static int srp_create_target_ib(struct srp_target_port *target) ...@@ -231,14 +231,16 @@ static int srp_create_target_ib(struct srp_target_port *target)
return -ENOMEM; return -ENOMEM;
recv_cq = ib_create_cq(target->srp_host->srp_dev->dev, recv_cq = ib_create_cq(target->srp_host->srp_dev->dev,
srp_recv_completion, NULL, target, SRP_RQ_SIZE, 0); srp_recv_completion, NULL, target, SRP_RQ_SIZE,
target->comp_vector);
if (IS_ERR(recv_cq)) { if (IS_ERR(recv_cq)) {
ret = PTR_ERR(recv_cq); ret = PTR_ERR(recv_cq);
goto err; goto err;
} }
send_cq = ib_create_cq(target->srp_host->srp_dev->dev, send_cq = ib_create_cq(target->srp_host->srp_dev->dev,
srp_send_completion, NULL, target, SRP_SQ_SIZE, 0); srp_send_completion, NULL, target, SRP_SQ_SIZE,
target->comp_vector);
if (IS_ERR(send_cq)) { if (IS_ERR(send_cq)) {
ret = PTR_ERR(send_cq); ret = PTR_ERR(send_cq);
goto err_recv_cq; goto err_recv_cq;
...@@ -1898,6 +1900,14 @@ static ssize_t show_local_ib_device(struct device *dev, ...@@ -1898,6 +1900,14 @@ static ssize_t show_local_ib_device(struct device *dev,
return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name); return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
} }
static ssize_t show_comp_vector(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct srp_target_port *target = host_to_target(class_to_shost(dev));
return sprintf(buf, "%d\n", target->comp_vector);
}
static ssize_t show_cmd_sg_entries(struct device *dev, static ssize_t show_cmd_sg_entries(struct device *dev,
struct device_attribute *attr, char *buf) struct device_attribute *attr, char *buf)
{ {
...@@ -1924,6 +1934,7 @@ static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL); ...@@ -1924,6 +1934,7 @@ static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL); static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL); static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL); static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL); static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL); static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
...@@ -1938,6 +1949,7 @@ static struct device_attribute *srp_host_attrs[] = { ...@@ -1938,6 +1949,7 @@ static struct device_attribute *srp_host_attrs[] = {
&dev_attr_zero_req_lim, &dev_attr_zero_req_lim,
&dev_attr_local_ib_port, &dev_attr_local_ib_port,
&dev_attr_local_ib_device, &dev_attr_local_ib_device,
&dev_attr_comp_vector,
&dev_attr_cmd_sg_entries, &dev_attr_cmd_sg_entries,
&dev_attr_allow_ext_sg, &dev_attr_allow_ext_sg,
NULL NULL
...@@ -2061,6 +2073,7 @@ enum { ...@@ -2061,6 +2073,7 @@ enum {
SRP_OPT_CMD_SG_ENTRIES = 1 << 9, SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
SRP_OPT_ALLOW_EXT_SG = 1 << 10, SRP_OPT_ALLOW_EXT_SG = 1 << 10,
SRP_OPT_SG_TABLESIZE = 1 << 11, SRP_OPT_SG_TABLESIZE = 1 << 11,
SRP_OPT_COMP_VECTOR = 1 << 12,
SRP_OPT_ALL = (SRP_OPT_ID_EXT | SRP_OPT_ALL = (SRP_OPT_ID_EXT |
SRP_OPT_IOC_GUID | SRP_OPT_IOC_GUID |
SRP_OPT_DGID | SRP_OPT_DGID |
...@@ -2081,6 +2094,7 @@ static const match_table_t srp_opt_tokens = { ...@@ -2081,6 +2094,7 @@ static const match_table_t srp_opt_tokens = {
{ SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" }, { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
{ SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" }, { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
{ SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" }, { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
{ SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
{ SRP_OPT_ERR, NULL } { SRP_OPT_ERR, NULL }
}; };
...@@ -2236,6 +2250,14 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target) ...@@ -2236,6 +2250,14 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
target->sg_tablesize = token; target->sg_tablesize = token;
break; break;
case SRP_OPT_COMP_VECTOR:
if (match_int(args, &token) || token < 0) {
pr_warn("bad comp_vector parameter '%s'\n", p);
goto out;
}
target->comp_vector = token;
break;
default: default:
pr_warn("unknown parameter or missing value '%s' in target creation request\n", pr_warn("unknown parameter or missing value '%s' in target creation request\n",
p); p);
......
...@@ -156,6 +156,7 @@ struct srp_target_port { ...@@ -156,6 +156,7 @@ struct srp_target_port {
char target_name[32]; char target_name[32];
unsigned int scsi_id; unsigned int scsi_id;
unsigned int sg_tablesize; unsigned int sg_tablesize;
int comp_vector;
struct ib_sa_path_rec path; struct ib_sa_path_rec path;
__be16 orig_dgid[8]; __be16 orig_dgid[8];
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment