Commit 509c07bc authored by Bart Van Assche's avatar Bart Van Assche Committed by Christoph Hellwig

IB/srp: Separate target and channel variables

Changes in this patch:
- Move channel variables into a new structure (struct srp_rdma_ch).
- Add an srp_target_port pointer, 'lock' and 'comp_vector' members
  in struct srp_rdma_ch.
- Add code to initialize these three new member variables.
- Many boring "target->" into "ch->" changes.
- The cm_id and completion handler context pointers are now of type
  srp_rdma_ch * instead of srp_target_port *.
- Three kzalloc(a * b, f) calls have been changed into kcalloc(a, b, f)
  to avoid that this patch would trigger a checkpatch warning.
- Two casts from u64 into unsigned long long have been left out
  because these are superfluous. Since considerable time u64 is
  defined as unsigned long long for all architectures supported by
  the Linux kernel.
Signed-off-by: default avatarBart Van Assche <bvanassche@acm.org>
Acked-by: default avatarSagi Grimberg <sagig@mellanox.com>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
parent 747fe000
...@@ -125,8 +125,8 @@ MODULE_PARM_DESC(dev_loss_tmo, ...@@ -125,8 +125,8 @@ MODULE_PARM_DESC(dev_loss_tmo,
static void srp_add_one(struct ib_device *device); static void srp_add_one(struct ib_device *device);
static void srp_remove_one(struct ib_device *device); static void srp_remove_one(struct ib_device *device);
static void srp_recv_completion(struct ib_cq *cq, void *target_ptr); static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
static void srp_send_completion(struct ib_cq *cq, void *target_ptr); static void srp_send_completion(struct ib_cq *cq, void *ch_ptr);
static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event); static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
static struct scsi_transport_template *ib_srp_transport_template; static struct scsi_transport_template *ib_srp_transport_template;
...@@ -283,22 +283,23 @@ static int srp_init_qp(struct srp_target_port *target, ...@@ -283,22 +283,23 @@ static int srp_init_qp(struct srp_target_port *target,
return ret; return ret;
} }
static int srp_new_cm_id(struct srp_target_port *target) static int srp_new_cm_id(struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
struct ib_cm_id *new_cm_id; struct ib_cm_id *new_cm_id;
new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev, new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
srp_cm_handler, target); srp_cm_handler, ch);
if (IS_ERR(new_cm_id)) if (IS_ERR(new_cm_id))
return PTR_ERR(new_cm_id); return PTR_ERR(new_cm_id);
if (target->cm_id) if (ch->cm_id)
ib_destroy_cm_id(target->cm_id); ib_destroy_cm_id(ch->cm_id);
target->cm_id = new_cm_id; ch->cm_id = new_cm_id;
target->path.sgid = target->sgid; ch->path.sgid = target->sgid;
target->path.dgid = target->orig_dgid; ch->path.dgid = target->orig_dgid;
target->path.pkey = target->pkey; ch->path.pkey = target->pkey;
target->path.service_id = target->service_id; ch->path.service_id = target->service_id;
return 0; return 0;
} }
...@@ -447,8 +448,9 @@ static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target) ...@@ -447,8 +448,9 @@ static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
dev->max_pages_per_mr); dev->max_pages_per_mr);
} }
static int srp_create_target_ib(struct srp_target_port *target) static int srp_create_ch_ib(struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
struct srp_device *dev = target->srp_host->srp_dev; struct srp_device *dev = target->srp_host->srp_dev;
struct ib_qp_init_attr *init_attr; struct ib_qp_init_attr *init_attr;
struct ib_cq *recv_cq, *send_cq; struct ib_cq *recv_cq, *send_cq;
...@@ -462,15 +464,15 @@ static int srp_create_target_ib(struct srp_target_port *target) ...@@ -462,15 +464,15 @@ static int srp_create_target_ib(struct srp_target_port *target)
if (!init_attr) if (!init_attr)
return -ENOMEM; return -ENOMEM;
recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, target, recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, ch,
target->queue_size, target->comp_vector); target->queue_size, ch->comp_vector);
if (IS_ERR(recv_cq)) { if (IS_ERR(recv_cq)) {
ret = PTR_ERR(recv_cq); ret = PTR_ERR(recv_cq);
goto err; goto err;
} }
send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, target, send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, ch,
m * target->queue_size, target->comp_vector); m * target->queue_size, ch->comp_vector);
if (IS_ERR(send_cq)) { if (IS_ERR(send_cq)) {
ret = PTR_ERR(send_cq); ret = PTR_ERR(send_cq);
goto err_recv_cq; goto err_recv_cq;
...@@ -506,9 +508,9 @@ static int srp_create_target_ib(struct srp_target_port *target) ...@@ -506,9 +508,9 @@ static int srp_create_target_ib(struct srp_target_port *target)
"FR pool allocation failed (%d)\n", ret); "FR pool allocation failed (%d)\n", ret);
goto err_qp; goto err_qp;
} }
if (target->fr_pool) if (ch->fr_pool)
srp_destroy_fr_pool(target->fr_pool); srp_destroy_fr_pool(ch->fr_pool);
target->fr_pool = fr_pool; ch->fr_pool = fr_pool;
} else if (!dev->use_fast_reg && dev->has_fmr) { } else if (!dev->use_fast_reg && dev->has_fmr) {
fmr_pool = srp_alloc_fmr_pool(target); fmr_pool = srp_alloc_fmr_pool(target);
if (IS_ERR(fmr_pool)) { if (IS_ERR(fmr_pool)) {
...@@ -517,21 +519,21 @@ static int srp_create_target_ib(struct srp_target_port *target) ...@@ -517,21 +519,21 @@ static int srp_create_target_ib(struct srp_target_port *target)
"FMR pool allocation failed (%d)\n", ret); "FMR pool allocation failed (%d)\n", ret);
goto err_qp; goto err_qp;
} }
if (target->fmr_pool) if (ch->fmr_pool)
ib_destroy_fmr_pool(target->fmr_pool); ib_destroy_fmr_pool(ch->fmr_pool);
target->fmr_pool = fmr_pool; ch->fmr_pool = fmr_pool;
} }
if (target->qp) if (ch->qp)
ib_destroy_qp(target->qp); ib_destroy_qp(ch->qp);
if (target->recv_cq) if (ch->recv_cq)
ib_destroy_cq(target->recv_cq); ib_destroy_cq(ch->recv_cq);
if (target->send_cq) if (ch->send_cq)
ib_destroy_cq(target->send_cq); ib_destroy_cq(ch->send_cq);
target->qp = qp; ch->qp = qp;
target->recv_cq = recv_cq; ch->recv_cq = recv_cq;
target->send_cq = send_cq; ch->send_cq = send_cq;
kfree(init_attr); kfree(init_attr);
return 0; return 0;
...@@ -552,98 +554,102 @@ static int srp_create_target_ib(struct srp_target_port *target) ...@@ -552,98 +554,102 @@ static int srp_create_target_ib(struct srp_target_port *target)
/* /*
* Note: this function may be called without srp_alloc_iu_bufs() having been * Note: this function may be called without srp_alloc_iu_bufs() having been
* invoked. Hence the target->[rt]x_ring checks. * invoked. Hence the ch->[rt]x_ring checks.
*/ */
static void srp_free_target_ib(struct srp_target_port *target) static void srp_free_ch_ib(struct srp_target_port *target,
struct srp_rdma_ch *ch)
{ {
struct srp_device *dev = target->srp_host->srp_dev; struct srp_device *dev = target->srp_host->srp_dev;
int i; int i;
if (target->cm_id) { if (ch->cm_id) {
ib_destroy_cm_id(target->cm_id); ib_destroy_cm_id(ch->cm_id);
target->cm_id = NULL; ch->cm_id = NULL;
} }
if (dev->use_fast_reg) { if (dev->use_fast_reg) {
if (target->fr_pool) if (ch->fr_pool)
srp_destroy_fr_pool(target->fr_pool); srp_destroy_fr_pool(ch->fr_pool);
} else { } else {
if (target->fmr_pool) if (ch->fmr_pool)
ib_destroy_fmr_pool(target->fmr_pool); ib_destroy_fmr_pool(ch->fmr_pool);
} }
ib_destroy_qp(target->qp); ib_destroy_qp(ch->qp);
ib_destroy_cq(target->send_cq); ib_destroy_cq(ch->send_cq);
ib_destroy_cq(target->recv_cq); ib_destroy_cq(ch->recv_cq);
target->qp = NULL; ch->qp = NULL;
target->send_cq = target->recv_cq = NULL; ch->send_cq = ch->recv_cq = NULL;
if (target->rx_ring) { if (ch->rx_ring) {
for (i = 0; i < target->queue_size; ++i) for (i = 0; i < target->queue_size; ++i)
srp_free_iu(target->srp_host, target->rx_ring[i]); srp_free_iu(target->srp_host, ch->rx_ring[i]);
kfree(target->rx_ring); kfree(ch->rx_ring);
target->rx_ring = NULL; ch->rx_ring = NULL;
} }
if (target->tx_ring) { if (ch->tx_ring) {
for (i = 0; i < target->queue_size; ++i) for (i = 0; i < target->queue_size; ++i)
srp_free_iu(target->srp_host, target->tx_ring[i]); srp_free_iu(target->srp_host, ch->tx_ring[i]);
kfree(target->tx_ring); kfree(ch->tx_ring);
target->tx_ring = NULL; ch->tx_ring = NULL;
} }
} }
static void srp_path_rec_completion(int status, static void srp_path_rec_completion(int status,
struct ib_sa_path_rec *pathrec, struct ib_sa_path_rec *pathrec,
void *target_ptr) void *ch_ptr)
{ {
struct srp_target_port *target = target_ptr; struct srp_rdma_ch *ch = ch_ptr;
struct srp_target_port *target = ch->target;
target->status = status; ch->status = status;
if (status) if (status)
shost_printk(KERN_ERR, target->scsi_host, shost_printk(KERN_ERR, target->scsi_host,
PFX "Got failed path rec status %d\n", status); PFX "Got failed path rec status %d\n", status);
else else
target->path = *pathrec; ch->path = *pathrec;
complete(&target->done); complete(&ch->done);
} }
static int srp_lookup_path(struct srp_target_port *target) static int srp_lookup_path(struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
int ret; int ret;
target->path.numb_path = 1; ch->path.numb_path = 1;
init_completion(&target->done); init_completion(&ch->done);
target->path_query_id = ib_sa_path_rec_get(&srp_sa_client, ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
target->srp_host->srp_dev->dev, target->srp_host->srp_dev->dev,
target->srp_host->port, target->srp_host->port,
&target->path, &ch->path,
IB_SA_PATH_REC_SERVICE_ID | IB_SA_PATH_REC_SERVICE_ID |
IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_DGID |
IB_SA_PATH_REC_SGID | IB_SA_PATH_REC_SGID |
IB_SA_PATH_REC_NUMB_PATH | IB_SA_PATH_REC_NUMB_PATH |
IB_SA_PATH_REC_PKEY, IB_SA_PATH_REC_PKEY,
SRP_PATH_REC_TIMEOUT_MS, SRP_PATH_REC_TIMEOUT_MS,
GFP_KERNEL, GFP_KERNEL,
srp_path_rec_completion, srp_path_rec_completion,
target, &target->path_query); ch, &ch->path_query);
if (target->path_query_id < 0) if (ch->path_query_id < 0)
return target->path_query_id; return ch->path_query_id;
ret = wait_for_completion_interruptible(&target->done); ret = wait_for_completion_interruptible(&ch->done);
if (ret < 0) if (ret < 0)
return ret; return ret;
if (target->status < 0) if (ch->status < 0)
shost_printk(KERN_WARNING, target->scsi_host, shost_printk(KERN_WARNING, target->scsi_host,
PFX "Path record query failed\n"); PFX "Path record query failed\n");
return target->status; return ch->status;
} }
static int srp_send_req(struct srp_target_port *target) static int srp_send_req(struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
struct { struct {
struct ib_cm_req_param param; struct ib_cm_req_param param;
struct srp_login_req priv; struct srp_login_req priv;
...@@ -654,11 +660,11 @@ static int srp_send_req(struct srp_target_port *target) ...@@ -654,11 +660,11 @@ static int srp_send_req(struct srp_target_port *target)
if (!req) if (!req)
return -ENOMEM; return -ENOMEM;
req->param.primary_path = &target->path; req->param.primary_path = &ch->path;
req->param.alternate_path = NULL; req->param.alternate_path = NULL;
req->param.service_id = target->service_id; req->param.service_id = target->service_id;
req->param.qp_num = target->qp->qp_num; req->param.qp_num = ch->qp->qp_num;
req->param.qp_type = target->qp->qp_type; req->param.qp_type = ch->qp->qp_type;
req->param.private_data = &req->priv; req->param.private_data = &req->priv;
req->param.private_data_len = sizeof req->priv; req->param.private_data_len = sizeof req->priv;
req->param.flow_control = 1; req->param.flow_control = 1;
...@@ -722,7 +728,7 @@ static int srp_send_req(struct srp_target_port *target) ...@@ -722,7 +728,7 @@ static int srp_send_req(struct srp_target_port *target)
&target->srp_host->srp_dev->dev->node_guid, 8); &target->srp_host->srp_dev->dev->node_guid, 8);
} }
status = ib_send_cm_req(target->cm_id, &req->param); status = ib_send_cm_req(ch->cm_id, &req->param);
kfree(req); kfree(req);
...@@ -763,28 +769,31 @@ static bool srp_change_conn_state(struct srp_target_port *target, ...@@ -763,28 +769,31 @@ static bool srp_change_conn_state(struct srp_target_port *target,
static void srp_disconnect_target(struct srp_target_port *target) static void srp_disconnect_target(struct srp_target_port *target)
{ {
struct srp_rdma_ch *ch = &target->ch;
if (srp_change_conn_state(target, false)) { if (srp_change_conn_state(target, false)) {
/* XXX should send SRP_I_LOGOUT request */ /* XXX should send SRP_I_LOGOUT request */
if (ib_send_cm_dreq(target->cm_id, NULL, 0)) { if (ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
shost_printk(KERN_DEBUG, target->scsi_host, shost_printk(KERN_DEBUG, target->scsi_host,
PFX "Sending CM DREQ failed\n"); PFX "Sending CM DREQ failed\n");
} }
} }
} }
static void srp_free_req_data(struct srp_target_port *target) static void srp_free_req_data(struct srp_target_port *target,
struct srp_rdma_ch *ch)
{ {
struct srp_device *dev = target->srp_host->srp_dev; struct srp_device *dev = target->srp_host->srp_dev;
struct ib_device *ibdev = dev->dev; struct ib_device *ibdev = dev->dev;
struct srp_request *req; struct srp_request *req;
int i; int i;
if (!target->req_ring) if (!ch->req_ring)
return; return;
for (i = 0; i < target->req_ring_size; ++i) { for (i = 0; i < target->req_ring_size; ++i) {
req = &target->req_ring[i]; req = &ch->req_ring[i];
if (dev->use_fast_reg) if (dev->use_fast_reg)
kfree(req->fr_list); kfree(req->fr_list);
else else
...@@ -798,12 +807,13 @@ static void srp_free_req_data(struct srp_target_port *target) ...@@ -798,12 +807,13 @@ static void srp_free_req_data(struct srp_target_port *target)
kfree(req->indirect_desc); kfree(req->indirect_desc);
} }
kfree(target->req_ring); kfree(ch->req_ring);
target->req_ring = NULL; ch->req_ring = NULL;
} }
static int srp_alloc_req_data(struct srp_target_port *target) static int srp_alloc_req_data(struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
struct srp_device *srp_dev = target->srp_host->srp_dev; struct srp_device *srp_dev = target->srp_host->srp_dev;
struct ib_device *ibdev = srp_dev->dev; struct ib_device *ibdev = srp_dev->dev;
struct srp_request *req; struct srp_request *req;
...@@ -811,15 +821,15 @@ static int srp_alloc_req_data(struct srp_target_port *target) ...@@ -811,15 +821,15 @@ static int srp_alloc_req_data(struct srp_target_port *target)
dma_addr_t dma_addr; dma_addr_t dma_addr;
int i, ret = -ENOMEM; int i, ret = -ENOMEM;
INIT_LIST_HEAD(&target->free_reqs); INIT_LIST_HEAD(&ch->free_reqs);
target->req_ring = kzalloc(target->req_ring_size * ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
sizeof(*target->req_ring), GFP_KERNEL); GFP_KERNEL);
if (!target->req_ring) if (!ch->req_ring)
goto out; goto out;
for (i = 0; i < target->req_ring_size; ++i) { for (i = 0; i < target->req_ring_size; ++i) {
req = &target->req_ring[i]; req = &ch->req_ring[i];
mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *), mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
GFP_KERNEL); GFP_KERNEL);
if (!mr_list) if (!mr_list)
...@@ -844,7 +854,7 @@ static int srp_alloc_req_data(struct srp_target_port *target) ...@@ -844,7 +854,7 @@ static int srp_alloc_req_data(struct srp_target_port *target)
req->indirect_dma_addr = dma_addr; req->indirect_dma_addr = dma_addr;
req->index = i; req->index = i;
list_add_tail(&req->list, &target->free_reqs); list_add_tail(&req->list, &ch->free_reqs);
} }
ret = 0; ret = 0;
...@@ -869,6 +879,8 @@ static void srp_del_scsi_host_attr(struct Scsi_Host *shost) ...@@ -869,6 +879,8 @@ static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
static void srp_remove_target(struct srp_target_port *target) static void srp_remove_target(struct srp_target_port *target)
{ {
struct srp_rdma_ch *ch = &target->ch;
WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
srp_del_scsi_host_attr(target->scsi_host); srp_del_scsi_host_attr(target->scsi_host);
...@@ -877,10 +889,10 @@ static void srp_remove_target(struct srp_target_port *target) ...@@ -877,10 +889,10 @@ static void srp_remove_target(struct srp_target_port *target)
scsi_remove_host(target->scsi_host); scsi_remove_host(target->scsi_host);
srp_stop_rport_timers(target->rport); srp_stop_rport_timers(target->rport);
srp_disconnect_target(target); srp_disconnect_target(target);
srp_free_target_ib(target); srp_free_ch_ib(target, ch);
cancel_work_sync(&target->tl_err_work); cancel_work_sync(&target->tl_err_work);
srp_rport_put(target->rport); srp_rport_put(target->rport);
srp_free_req_data(target); srp_free_req_data(target, ch);
spin_lock(&target->srp_host->target_lock); spin_lock(&target->srp_host->target_lock);
list_del(&target->list); list_del(&target->list);
...@@ -906,24 +918,25 @@ static void srp_rport_delete(struct srp_rport *rport) ...@@ -906,24 +918,25 @@ static void srp_rport_delete(struct srp_rport *rport)
srp_queue_remove_work(target); srp_queue_remove_work(target);
} }
static int srp_connect_target(struct srp_target_port *target) static int srp_connect_ch(struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
int ret; int ret;
WARN_ON_ONCE(target->connected); WARN_ON_ONCE(target->connected);
target->qp_in_error = false; target->qp_in_error = false;
ret = srp_lookup_path(target); ret = srp_lookup_path(ch);
if (ret) if (ret)
return ret; return ret;
while (1) { while (1) {
init_completion(&target->done); init_completion(&ch->done);
ret = srp_send_req(target); ret = srp_send_req(ch);
if (ret) if (ret)
return ret; return ret;
ret = wait_for_completion_interruptible(&target->done); ret = wait_for_completion_interruptible(&ch->done);
if (ret < 0) if (ret < 0)
return ret; return ret;
...@@ -933,13 +946,13 @@ static int srp_connect_target(struct srp_target_port *target) ...@@ -933,13 +946,13 @@ static int srp_connect_target(struct srp_target_port *target)
* back, or SRP_DLID_REDIRECT if we get a lid/qp * back, or SRP_DLID_REDIRECT if we get a lid/qp
* redirect REJ back. * redirect REJ back.
*/ */
switch (target->status) { switch (ch->status) {
case 0: case 0:
srp_change_conn_state(target, true); srp_change_conn_state(target, true);
return 0; return 0;
case SRP_PORT_REDIRECT: case SRP_PORT_REDIRECT:
ret = srp_lookup_path(target); ret = srp_lookup_path(ch);
if (ret) if (ret)
return ret; return ret;
break; break;
...@@ -950,16 +963,16 @@ static int srp_connect_target(struct srp_target_port *target) ...@@ -950,16 +963,16 @@ static int srp_connect_target(struct srp_target_port *target)
case SRP_STALE_CONN: case SRP_STALE_CONN:
shost_printk(KERN_ERR, target->scsi_host, PFX shost_printk(KERN_ERR, target->scsi_host, PFX
"giving up on stale connection\n"); "giving up on stale connection\n");
target->status = -ECONNRESET; ch->status = -ECONNRESET;
return target->status; return ch->status;
default: default:
return target->status; return ch->status;
} }
} }
} }
static int srp_inv_rkey(struct srp_target_port *target, u32 rkey) static int srp_inv_rkey(struct srp_rdma_ch *ch, u32 rkey)
{ {
struct ib_send_wr *bad_wr; struct ib_send_wr *bad_wr;
struct ib_send_wr wr = { struct ib_send_wr wr = {
...@@ -971,13 +984,14 @@ static int srp_inv_rkey(struct srp_target_port *target, u32 rkey) ...@@ -971,13 +984,14 @@ static int srp_inv_rkey(struct srp_target_port *target, u32 rkey)
.ex.invalidate_rkey = rkey, .ex.invalidate_rkey = rkey,
}; };
return ib_post_send(target->qp, &wr, &bad_wr); return ib_post_send(ch->qp, &wr, &bad_wr);
} }
static void srp_unmap_data(struct scsi_cmnd *scmnd, static void srp_unmap_data(struct scsi_cmnd *scmnd,
struct srp_target_port *target, struct srp_rdma_ch *ch,
struct srp_request *req) struct srp_request *req)
{ {
struct srp_target_port *target = ch->target;
struct srp_device *dev = target->srp_host->srp_dev; struct srp_device *dev = target->srp_host->srp_dev;
struct ib_device *ibdev = dev->dev; struct ib_device *ibdev = dev->dev;
int i, res; int i, res;
...@@ -991,7 +1005,7 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd, ...@@ -991,7 +1005,7 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd,
struct srp_fr_desc **pfr; struct srp_fr_desc **pfr;
for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) { for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
res = srp_inv_rkey(target, (*pfr)->mr->rkey); res = srp_inv_rkey(ch, (*pfr)->mr->rkey);
if (res < 0) { if (res < 0) {
shost_printk(KERN_ERR, target->scsi_host, PFX shost_printk(KERN_ERR, target->scsi_host, PFX
"Queueing INV WR for rkey %#x failed (%d)\n", "Queueing INV WR for rkey %#x failed (%d)\n",
...@@ -1001,7 +1015,7 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd, ...@@ -1001,7 +1015,7 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd,
} }
} }
if (req->nmdesc) if (req->nmdesc)
srp_fr_pool_put(target->fr_pool, req->fr_list, srp_fr_pool_put(ch->fr_pool, req->fr_list,
req->nmdesc); req->nmdesc);
} else { } else {
struct ib_pool_fmr **pfmr; struct ib_pool_fmr **pfmr;
...@@ -1016,7 +1030,7 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd, ...@@ -1016,7 +1030,7 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd,
/** /**
* srp_claim_req - Take ownership of the scmnd associated with a request. * srp_claim_req - Take ownership of the scmnd associated with a request.
* @target: SRP target port. * @ch: SRP RDMA channel.
* @req: SRP request. * @req: SRP request.
* @sdev: If not NULL, only take ownership for this SCSI device. * @sdev: If not NULL, only take ownership for this SCSI device.
* @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
...@@ -1025,14 +1039,14 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd, ...@@ -1025,14 +1039,14 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd,
* Return value: * Return value:
* Either NULL or a pointer to the SCSI command the caller became owner of. * Either NULL or a pointer to the SCSI command the caller became owner of.
*/ */
static struct scsi_cmnd *srp_claim_req(struct srp_target_port *target, static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
struct srp_request *req, struct srp_request *req,
struct scsi_device *sdev, struct scsi_device *sdev,
struct scsi_cmnd *scmnd) struct scsi_cmnd *scmnd)
{ {
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&target->lock, flags); spin_lock_irqsave(&ch->lock, flags);
if (req->scmnd && if (req->scmnd &&
(!sdev || req->scmnd->device == sdev) && (!sdev || req->scmnd->device == sdev) &&
(!scmnd || req->scmnd == scmnd)) { (!scmnd || req->scmnd == scmnd)) {
...@@ -1041,40 +1055,38 @@ static struct scsi_cmnd *srp_claim_req(struct srp_target_port *target, ...@@ -1041,40 +1055,38 @@ static struct scsi_cmnd *srp_claim_req(struct srp_target_port *target,
} else { } else {
scmnd = NULL; scmnd = NULL;
} }
spin_unlock_irqrestore(&target->lock, flags); spin_unlock_irqrestore(&ch->lock, flags);
return scmnd; return scmnd;
} }
/** /**
* srp_free_req() - Unmap data and add request to the free request list. * srp_free_req() - Unmap data and add request to the free request list.
* @target: SRP target port. * @ch: SRP RDMA channel.
* @req: Request to be freed. * @req: Request to be freed.
* @scmnd: SCSI command associated with @req. * @scmnd: SCSI command associated with @req.
* @req_lim_delta: Amount to be added to @target->req_lim. * @req_lim_delta: Amount to be added to @target->req_lim.
*/ */
static void srp_free_req(struct srp_target_port *target, static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
struct srp_request *req, struct scsi_cmnd *scmnd, struct scsi_cmnd *scmnd, s32 req_lim_delta)
s32 req_lim_delta)
{ {
unsigned long flags; unsigned long flags;
srp_unmap_data(scmnd, target, req); srp_unmap_data(scmnd, ch, req);
spin_lock_irqsave(&target->lock, flags); spin_lock_irqsave(&ch->lock, flags);
target->req_lim += req_lim_delta; ch->req_lim += req_lim_delta;
list_add_tail(&req->list, &target->free_reqs); list_add_tail(&req->list, &ch->free_reqs);
spin_unlock_irqrestore(&target->lock, flags); spin_unlock_irqrestore(&ch->lock, flags);
} }
static void srp_finish_req(struct srp_target_port *target, static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
struct srp_request *req, struct scsi_device *sdev, struct scsi_device *sdev, int result)
int result)
{ {
struct scsi_cmnd *scmnd = srp_claim_req(target, req, sdev, NULL); struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
if (scmnd) { if (scmnd) {
srp_free_req(target, req, scmnd, 0); srp_free_req(ch, req, scmnd, 0);
scmnd->result = result; scmnd->result = result;
scmnd->scsi_done(scmnd); scmnd->scsi_done(scmnd);
} }
...@@ -1083,6 +1095,7 @@ static void srp_finish_req(struct srp_target_port *target, ...@@ -1083,6 +1095,7 @@ static void srp_finish_req(struct srp_target_port *target,
static void srp_terminate_io(struct srp_rport *rport) static void srp_terminate_io(struct srp_rport *rport)
{ {
struct srp_target_port *target = rport->lld_data; struct srp_target_port *target = rport->lld_data;
struct srp_rdma_ch *ch = &target->ch;
struct Scsi_Host *shost = target->scsi_host; struct Scsi_Host *shost = target->scsi_host;
struct scsi_device *sdev; struct scsi_device *sdev;
int i; int i;
...@@ -1095,8 +1108,9 @@ static void srp_terminate_io(struct srp_rport *rport) ...@@ -1095,8 +1108,9 @@ static void srp_terminate_io(struct srp_rport *rport)
WARN_ON_ONCE(sdev->request_queue->request_fn_active); WARN_ON_ONCE(sdev->request_queue->request_fn_active);
for (i = 0; i < target->req_ring_size; ++i) { for (i = 0; i < target->req_ring_size; ++i) {
struct srp_request *req = &target->req_ring[i]; struct srp_request *req = &ch->req_ring[i];
srp_finish_req(target, req, NULL, DID_TRANSPORT_FAILFAST << 16);
srp_finish_req(ch, req, NULL, DID_TRANSPORT_FAILFAST << 16);
} }
} }
...@@ -1112,6 +1126,7 @@ static void srp_terminate_io(struct srp_rport *rport) ...@@ -1112,6 +1126,7 @@ static void srp_terminate_io(struct srp_rport *rport)
static int srp_rport_reconnect(struct srp_rport *rport) static int srp_rport_reconnect(struct srp_rport *rport)
{ {
struct srp_target_port *target = rport->lld_data; struct srp_target_port *target = rport->lld_data;
struct srp_rdma_ch *ch = &target->ch;
int i, ret; int i, ret;
srp_disconnect_target(target); srp_disconnect_target(target);
...@@ -1124,11 +1139,12 @@ static int srp_rport_reconnect(struct srp_rport *rport) ...@@ -1124,11 +1139,12 @@ static int srp_rport_reconnect(struct srp_rport *rport)
* case things are really fouled up. Doing so also ensures that all CM * case things are really fouled up. Doing so also ensures that all CM
* callbacks will have finished before a new QP is allocated. * callbacks will have finished before a new QP is allocated.
*/ */
ret = srp_new_cm_id(target); ret = srp_new_cm_id(ch);
for (i = 0; i < target->req_ring_size; ++i) { for (i = 0; i < target->req_ring_size; ++i) {
struct srp_request *req = &target->req_ring[i]; struct srp_request *req = &ch->req_ring[i];
srp_finish_req(target, req, NULL, DID_RESET << 16);
srp_finish_req(ch, req, NULL, DID_RESET << 16);
} }
/* /*
...@@ -1136,14 +1152,14 @@ static int srp_rport_reconnect(struct srp_rport *rport) ...@@ -1136,14 +1152,14 @@ static int srp_rport_reconnect(struct srp_rport *rport)
* QP. This guarantees that all callback functions for the old QP have * QP. This guarantees that all callback functions for the old QP have
* finished before any send requests are posted on the new QP. * finished before any send requests are posted on the new QP.
*/ */
ret += srp_create_target_ib(target); ret += srp_create_ch_ib(ch);
INIT_LIST_HEAD(&target->free_tx); INIT_LIST_HEAD(&ch->free_tx);
for (i = 0; i < target->queue_size; ++i) for (i = 0; i < target->queue_size; ++i)
list_add(&target->tx_ring[i]->list, &target->free_tx); list_add(&ch->tx_ring[i]->list, &ch->free_tx);
if (ret == 0) if (ret == 0)
ret = srp_connect_target(target); ret = srp_connect_ch(ch);
if (ret == 0) if (ret == 0)
shost_printk(KERN_INFO, target->scsi_host, shost_printk(KERN_INFO, target->scsi_host,
...@@ -1167,12 +1183,12 @@ static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr, ...@@ -1167,12 +1183,12 @@ static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
} }
static int srp_map_finish_fmr(struct srp_map_state *state, static int srp_map_finish_fmr(struct srp_map_state *state,
struct srp_target_port *target) struct srp_rdma_ch *ch)
{ {
struct ib_pool_fmr *fmr; struct ib_pool_fmr *fmr;
u64 io_addr = 0; u64 io_addr = 0;
fmr = ib_fmr_pool_map_phys(target->fmr_pool, state->pages, fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
state->npages, io_addr); state->npages, io_addr);
if (IS_ERR(fmr)) if (IS_ERR(fmr))
return PTR_ERR(fmr); return PTR_ERR(fmr);
...@@ -1186,15 +1202,16 @@ static int srp_map_finish_fmr(struct srp_map_state *state, ...@@ -1186,15 +1202,16 @@ static int srp_map_finish_fmr(struct srp_map_state *state,
} }
static int srp_map_finish_fr(struct srp_map_state *state, static int srp_map_finish_fr(struct srp_map_state *state,
struct srp_target_port *target) struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
struct srp_device *dev = target->srp_host->srp_dev; struct srp_device *dev = target->srp_host->srp_dev;
struct ib_send_wr *bad_wr; struct ib_send_wr *bad_wr;
struct ib_send_wr wr; struct ib_send_wr wr;
struct srp_fr_desc *desc; struct srp_fr_desc *desc;
u32 rkey; u32 rkey;
desc = srp_fr_pool_get(target->fr_pool); desc = srp_fr_pool_get(ch->fr_pool);
if (!desc) if (!desc)
return -ENOMEM; return -ENOMEM;
...@@ -1223,12 +1240,13 @@ static int srp_map_finish_fr(struct srp_map_state *state, ...@@ -1223,12 +1240,13 @@ static int srp_map_finish_fr(struct srp_map_state *state,
srp_map_desc(state, state->base_dma_addr, state->dma_len, srp_map_desc(state, state->base_dma_addr, state->dma_len,
desc->mr->rkey); desc->mr->rkey);
return ib_post_send(target->qp, &wr, &bad_wr); return ib_post_send(ch->qp, &wr, &bad_wr);
} }
static int srp_finish_mapping(struct srp_map_state *state, static int srp_finish_mapping(struct srp_map_state *state,
struct srp_target_port *target) struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
int ret = 0; int ret = 0;
if (state->npages == 0) if (state->npages == 0)
...@@ -1239,8 +1257,8 @@ static int srp_finish_mapping(struct srp_map_state *state, ...@@ -1239,8 +1257,8 @@ static int srp_finish_mapping(struct srp_map_state *state,
target->rkey); target->rkey);
else else
ret = target->srp_host->srp_dev->use_fast_reg ? ret = target->srp_host->srp_dev->use_fast_reg ?
srp_map_finish_fr(state, target) : srp_map_finish_fr(state, ch) :
srp_map_finish_fmr(state, target); srp_map_finish_fmr(state, ch);
if (ret == 0) { if (ret == 0) {
state->npages = 0; state->npages = 0;
...@@ -1260,10 +1278,11 @@ static void srp_map_update_start(struct srp_map_state *state, ...@@ -1260,10 +1278,11 @@ static void srp_map_update_start(struct srp_map_state *state,
} }
static int srp_map_sg_entry(struct srp_map_state *state, static int srp_map_sg_entry(struct srp_map_state *state,
struct srp_target_port *target, struct srp_rdma_ch *ch,
struct scatterlist *sg, int sg_index, struct scatterlist *sg, int sg_index,
bool use_mr) bool use_mr)
{ {
struct srp_target_port *target = ch->target;
struct srp_device *dev = target->srp_host->srp_dev; struct srp_device *dev = target->srp_host->srp_dev;
struct ib_device *ibdev = dev->dev; struct ib_device *ibdev = dev->dev;
dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg); dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
...@@ -1292,7 +1311,7 @@ static int srp_map_sg_entry(struct srp_map_state *state, ...@@ -1292,7 +1311,7 @@ static int srp_map_sg_entry(struct srp_map_state *state,
*/ */
if ((!dev->use_fast_reg && dma_addr & ~dev->mr_page_mask) || if ((!dev->use_fast_reg && dma_addr & ~dev->mr_page_mask) ||
dma_len > dev->mr_max_size) { dma_len > dev->mr_max_size) {
ret = srp_finish_mapping(state, target); ret = srp_finish_mapping(state, ch);
if (ret) if (ret)
return ret; return ret;
...@@ -1313,7 +1332,7 @@ static int srp_map_sg_entry(struct srp_map_state *state, ...@@ -1313,7 +1332,7 @@ static int srp_map_sg_entry(struct srp_map_state *state,
while (dma_len) { while (dma_len) {
unsigned offset = dma_addr & ~dev->mr_page_mask; unsigned offset = dma_addr & ~dev->mr_page_mask;
if (state->npages == dev->max_pages_per_mr || offset != 0) { if (state->npages == dev->max_pages_per_mr || offset != 0) {
ret = srp_finish_mapping(state, target); ret = srp_finish_mapping(state, ch);
if (ret) if (ret)
return ret; return ret;
...@@ -1337,17 +1356,18 @@ static int srp_map_sg_entry(struct srp_map_state *state, ...@@ -1337,17 +1356,18 @@ static int srp_map_sg_entry(struct srp_map_state *state,
*/ */
ret = 0; ret = 0;
if (len != dev->mr_page_size) { if (len != dev->mr_page_size) {
ret = srp_finish_mapping(state, target); ret = srp_finish_mapping(state, ch);
if (!ret) if (!ret)
srp_map_update_start(state, NULL, 0, 0); srp_map_update_start(state, NULL, 0, 0);
} }
return ret; return ret;
} }
static int srp_map_sg(struct srp_map_state *state, static int srp_map_sg(struct srp_map_state *state, struct srp_rdma_ch *ch,
struct srp_target_port *target, struct srp_request *req, struct srp_request *req, struct scatterlist *scat,
struct scatterlist *scat, int count) int count)
{ {
struct srp_target_port *target = ch->target;
struct srp_device *dev = target->srp_host->srp_dev; struct srp_device *dev = target->srp_host->srp_dev;
struct ib_device *ibdev = dev->dev; struct ib_device *ibdev = dev->dev;
struct scatterlist *sg; struct scatterlist *sg;
...@@ -1358,14 +1378,14 @@ static int srp_map_sg(struct srp_map_state *state, ...@@ -1358,14 +1378,14 @@ static int srp_map_sg(struct srp_map_state *state,
state->pages = req->map_page; state->pages = req->map_page;
if (dev->use_fast_reg) { if (dev->use_fast_reg) {
state->next_fr = req->fr_list; state->next_fr = req->fr_list;
use_mr = !!target->fr_pool; use_mr = !!ch->fr_pool;
} else { } else {
state->next_fmr = req->fmr_list; state->next_fmr = req->fmr_list;
use_mr = !!target->fmr_pool; use_mr = !!ch->fmr_pool;
} }
for_each_sg(scat, sg, count, i) { for_each_sg(scat, sg, count, i) {
if (srp_map_sg_entry(state, target, sg, i, use_mr)) { if (srp_map_sg_entry(state, ch, sg, i, use_mr)) {
/* /*
* Memory registration failed, so backtrack to the * Memory registration failed, so backtrack to the
* first unmapped entry and continue on without using * first unmapped entry and continue on without using
...@@ -1387,7 +1407,7 @@ static int srp_map_sg(struct srp_map_state *state, ...@@ -1387,7 +1407,7 @@ static int srp_map_sg(struct srp_map_state *state,
} }
} }
if (use_mr && srp_finish_mapping(state, target)) if (use_mr && srp_finish_mapping(state, ch))
goto backtrack; goto backtrack;
req->nmdesc = state->nmdesc; req->nmdesc = state->nmdesc;
...@@ -1395,9 +1415,10 @@ static int srp_map_sg(struct srp_map_state *state, ...@@ -1395,9 +1415,10 @@ static int srp_map_sg(struct srp_map_state *state,
return 0; return 0;
} }
static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_target_port *target, static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
struct srp_request *req) struct srp_request *req)
{ {
struct srp_target_port *target = ch->target;
struct scatterlist *scat; struct scatterlist *scat;
struct srp_cmd *cmd = req->cmd->buf; struct srp_cmd *cmd = req->cmd->buf;
int len, nents, count; int len, nents, count;
...@@ -1459,7 +1480,7 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_target_port *target, ...@@ -1459,7 +1480,7 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_target_port *target,
target->indirect_size, DMA_TO_DEVICE); target->indirect_size, DMA_TO_DEVICE);
memset(&state, 0, sizeof(state)); memset(&state, 0, sizeof(state));
srp_map_sg(&state, target, req, scat, count); srp_map_sg(&state, ch, req, scat, count);
/* We've mapped the request, now pull as much of the indirect /* We've mapped the request, now pull as much of the indirect
* descriptor table as we can into the command buffer. If this * descriptor table as we can into the command buffer. If this
...@@ -1520,20 +1541,20 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_target_port *target, ...@@ -1520,20 +1541,20 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_target_port *target,
/* /*
* Return an IU and possible credit to the free pool * Return an IU and possible credit to the free pool
*/ */
static void srp_put_tx_iu(struct srp_target_port *target, struct srp_iu *iu, static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
enum srp_iu_type iu_type) enum srp_iu_type iu_type)
{ {
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&target->lock, flags); spin_lock_irqsave(&ch->lock, flags);
list_add(&iu->list, &target->free_tx); list_add(&iu->list, &ch->free_tx);
if (iu_type != SRP_IU_RSP) if (iu_type != SRP_IU_RSP)
++target->req_lim; ++ch->req_lim;
spin_unlock_irqrestore(&target->lock, flags); spin_unlock_irqrestore(&ch->lock, flags);
} }
/* /*
* Must be called with target->lock held to protect req_lim and free_tx. * Must be called with ch->lock held to protect req_lim and free_tx.
* If IU is not sent, it must be returned using srp_put_tx_iu(). * If IU is not sent, it must be returned using srp_put_tx_iu().
* *
* Note: * Note:
...@@ -1545,35 +1566,36 @@ static void srp_put_tx_iu(struct srp_target_port *target, struct srp_iu *iu, ...@@ -1545,35 +1566,36 @@ static void srp_put_tx_iu(struct srp_target_port *target, struct srp_iu *iu,
* - SRP_IU_RSP: 1, since a conforming SRP target never sends more than * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
* one unanswered SRP request to an initiator. * one unanswered SRP request to an initiator.
*/ */
static struct srp_iu *__srp_get_tx_iu(struct srp_target_port *target, static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
enum srp_iu_type iu_type) enum srp_iu_type iu_type)
{ {
struct srp_target_port *target = ch->target;
s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE; s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
struct srp_iu *iu; struct srp_iu *iu;
srp_send_completion(target->send_cq, target); srp_send_completion(ch->send_cq, ch);
if (list_empty(&target->free_tx)) if (list_empty(&ch->free_tx))
return NULL; return NULL;
/* Initiator responses to target requests do not consume credits */ /* Initiator responses to target requests do not consume credits */
if (iu_type != SRP_IU_RSP) { if (iu_type != SRP_IU_RSP) {
if (target->req_lim <= rsv) { if (ch->req_lim <= rsv) {
++target->zero_req_lim; ++target->zero_req_lim;
return NULL; return NULL;
} }
--target->req_lim; --ch->req_lim;
} }
iu = list_first_entry(&target->free_tx, struct srp_iu, list); iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
list_del(&iu->list); list_del(&iu->list);
return iu; return iu;
} }
static int srp_post_send(struct srp_target_port *target, static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
struct srp_iu *iu, int len)
{ {
struct srp_target_port *target = ch->target;
struct ib_sge list; struct ib_sge list;
struct ib_send_wr wr, *bad_wr; struct ib_send_wr wr, *bad_wr;
...@@ -1588,11 +1610,12 @@ static int srp_post_send(struct srp_target_port *target, ...@@ -1588,11 +1610,12 @@ static int srp_post_send(struct srp_target_port *target,
wr.opcode = IB_WR_SEND; wr.opcode = IB_WR_SEND;
wr.send_flags = IB_SEND_SIGNALED; wr.send_flags = IB_SEND_SIGNALED;
return ib_post_send(target->qp, &wr, &bad_wr); return ib_post_send(ch->qp, &wr, &bad_wr);
} }
static int srp_post_recv(struct srp_target_port *target, struct srp_iu *iu) static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
{ {
struct srp_target_port *target = ch->target;
struct ib_recv_wr wr, *bad_wr; struct ib_recv_wr wr, *bad_wr;
struct ib_sge list; struct ib_sge list;
...@@ -1605,35 +1628,36 @@ static int srp_post_recv(struct srp_target_port *target, struct srp_iu *iu) ...@@ -1605,35 +1628,36 @@ static int srp_post_recv(struct srp_target_port *target, struct srp_iu *iu)
wr.sg_list = &list; wr.sg_list = &list;
wr.num_sge = 1; wr.num_sge = 1;
return ib_post_recv(target->qp, &wr, &bad_wr); return ib_post_recv(ch->qp, &wr, &bad_wr);
} }
static void srp_process_rsp(struct srp_target_port *target, struct srp_rsp *rsp) static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
{ {
struct srp_target_port *target = ch->target;
struct srp_request *req; struct srp_request *req;
struct scsi_cmnd *scmnd; struct scsi_cmnd *scmnd;
unsigned long flags; unsigned long flags;
if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) { if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
spin_lock_irqsave(&target->lock, flags); spin_lock_irqsave(&ch->lock, flags);
target->req_lim += be32_to_cpu(rsp->req_lim_delta); ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
spin_unlock_irqrestore(&target->lock, flags); spin_unlock_irqrestore(&ch->lock, flags);
target->tsk_mgmt_status = -1; ch->tsk_mgmt_status = -1;
if (be32_to_cpu(rsp->resp_data_len) >= 4) if (be32_to_cpu(rsp->resp_data_len) >= 4)
target->tsk_mgmt_status = rsp->data[3]; ch->tsk_mgmt_status = rsp->data[3];
complete(&target->tsk_mgmt_done); complete(&ch->tsk_mgmt_done);
} else { } else {
req = &target->req_ring[rsp->tag]; req = &ch->req_ring[rsp->tag];
scmnd = srp_claim_req(target, req, NULL, NULL); scmnd = srp_claim_req(ch, req, NULL, NULL);
if (!scmnd) { if (!scmnd) {
shost_printk(KERN_ERR, target->scsi_host, shost_printk(KERN_ERR, target->scsi_host,
"Null scmnd for RSP w/tag %016llx\n", "Null scmnd for RSP w/tag %016llx\n",
(unsigned long long) rsp->tag); (unsigned long long) rsp->tag);
spin_lock_irqsave(&target->lock, flags); spin_lock_irqsave(&ch->lock, flags);
target->req_lim += be32_to_cpu(rsp->req_lim_delta); ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
spin_unlock_irqrestore(&target->lock, flags); spin_unlock_irqrestore(&ch->lock, flags);
return; return;
} }
...@@ -1655,7 +1679,7 @@ static void srp_process_rsp(struct srp_target_port *target, struct srp_rsp *rsp) ...@@ -1655,7 +1679,7 @@ static void srp_process_rsp(struct srp_target_port *target, struct srp_rsp *rsp)
else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER)) else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt)); scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
srp_free_req(target, req, scmnd, srp_free_req(ch, req, scmnd,
be32_to_cpu(rsp->req_lim_delta)); be32_to_cpu(rsp->req_lim_delta));
scmnd->host_scribble = NULL; scmnd->host_scribble = NULL;
...@@ -1663,18 +1687,19 @@ static void srp_process_rsp(struct srp_target_port *target, struct srp_rsp *rsp) ...@@ -1663,18 +1687,19 @@ static void srp_process_rsp(struct srp_target_port *target, struct srp_rsp *rsp)
} }
} }
static int srp_response_common(struct srp_target_port *target, s32 req_delta, static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
void *rsp, int len) void *rsp, int len)
{ {
struct srp_target_port *target = ch->target;
struct ib_device *dev = target->srp_host->srp_dev->dev; struct ib_device *dev = target->srp_host->srp_dev->dev;
unsigned long flags; unsigned long flags;
struct srp_iu *iu; struct srp_iu *iu;
int err; int err;
spin_lock_irqsave(&target->lock, flags); spin_lock_irqsave(&ch->lock, flags);
target->req_lim += req_delta; ch->req_lim += req_delta;
iu = __srp_get_tx_iu(target, SRP_IU_RSP); iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
spin_unlock_irqrestore(&target->lock, flags); spin_unlock_irqrestore(&ch->lock, flags);
if (!iu) { if (!iu) {
shost_printk(KERN_ERR, target->scsi_host, PFX shost_printk(KERN_ERR, target->scsi_host, PFX
...@@ -1686,17 +1711,17 @@ static int srp_response_common(struct srp_target_port *target, s32 req_delta, ...@@ -1686,17 +1711,17 @@ static int srp_response_common(struct srp_target_port *target, s32 req_delta,
memcpy(iu->buf, rsp, len); memcpy(iu->buf, rsp, len);
ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE); ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
err = srp_post_send(target, iu, len); err = srp_post_send(ch, iu, len);
if (err) { if (err) {
shost_printk(KERN_ERR, target->scsi_host, PFX shost_printk(KERN_ERR, target->scsi_host, PFX
"unable to post response: %d\n", err); "unable to post response: %d\n", err);
srp_put_tx_iu(target, iu, SRP_IU_RSP); srp_put_tx_iu(ch, iu, SRP_IU_RSP);
} }
return err; return err;
} }
static void srp_process_cred_req(struct srp_target_port *target, static void srp_process_cred_req(struct srp_rdma_ch *ch,
struct srp_cred_req *req) struct srp_cred_req *req)
{ {
struct srp_cred_rsp rsp = { struct srp_cred_rsp rsp = {
...@@ -1705,14 +1730,15 @@ static void srp_process_cred_req(struct srp_target_port *target, ...@@ -1705,14 +1730,15 @@ static void srp_process_cred_req(struct srp_target_port *target,
}; };
s32 delta = be32_to_cpu(req->req_lim_delta); s32 delta = be32_to_cpu(req->req_lim_delta);
if (srp_response_common(target, delta, &rsp, sizeof rsp)) if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
shost_printk(KERN_ERR, target->scsi_host, PFX shost_printk(KERN_ERR, ch->target->scsi_host, PFX
"problems processing SRP_CRED_REQ\n"); "problems processing SRP_CRED_REQ\n");
} }
static void srp_process_aer_req(struct srp_target_port *target, static void srp_process_aer_req(struct srp_rdma_ch *ch,
struct srp_aer_req *req) struct srp_aer_req *req)
{ {
struct srp_target_port *target = ch->target;
struct srp_aer_rsp rsp = { struct srp_aer_rsp rsp = {
.opcode = SRP_AER_RSP, .opcode = SRP_AER_RSP,
.tag = req->tag, .tag = req->tag,
...@@ -1722,19 +1748,20 @@ static void srp_process_aer_req(struct srp_target_port *target, ...@@ -1722,19 +1748,20 @@ static void srp_process_aer_req(struct srp_target_port *target,
shost_printk(KERN_ERR, target->scsi_host, PFX shost_printk(KERN_ERR, target->scsi_host, PFX
"ignoring AER for LUN %llu\n", be64_to_cpu(req->lun)); "ignoring AER for LUN %llu\n", be64_to_cpu(req->lun));
if (srp_response_common(target, delta, &rsp, sizeof rsp)) if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
shost_printk(KERN_ERR, target->scsi_host, PFX shost_printk(KERN_ERR, target->scsi_host, PFX
"problems processing SRP_AER_REQ\n"); "problems processing SRP_AER_REQ\n");
} }
static void srp_handle_recv(struct srp_target_port *target, struct ib_wc *wc) static void srp_handle_recv(struct srp_rdma_ch *ch, struct ib_wc *wc)
{ {
struct srp_target_port *target = ch->target;
struct ib_device *dev = target->srp_host->srp_dev->dev; struct ib_device *dev = target->srp_host->srp_dev->dev;
struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id; struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
int res; int res;
u8 opcode; u8 opcode;
ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_ti_iu_len, ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
DMA_FROM_DEVICE); DMA_FROM_DEVICE);
opcode = *(u8 *) iu->buf; opcode = *(u8 *) iu->buf;
...@@ -1748,15 +1775,15 @@ static void srp_handle_recv(struct srp_target_port *target, struct ib_wc *wc) ...@@ -1748,15 +1775,15 @@ static void srp_handle_recv(struct srp_target_port *target, struct ib_wc *wc)
switch (opcode) { switch (opcode) {
case SRP_RSP: case SRP_RSP:
srp_process_rsp(target, iu->buf); srp_process_rsp(ch, iu->buf);
break; break;
case SRP_CRED_REQ: case SRP_CRED_REQ:
srp_process_cred_req(target, iu->buf); srp_process_cred_req(ch, iu->buf);
break; break;
case SRP_AER_REQ: case SRP_AER_REQ:
srp_process_aer_req(target, iu->buf); srp_process_aer_req(ch, iu->buf);
break; break;
case SRP_T_LOGOUT: case SRP_T_LOGOUT:
...@@ -1771,10 +1798,10 @@ static void srp_handle_recv(struct srp_target_port *target, struct ib_wc *wc) ...@@ -1771,10 +1798,10 @@ static void srp_handle_recv(struct srp_target_port *target, struct ib_wc *wc)
break; break;
} }
ib_dma_sync_single_for_device(dev, iu->dma, target->max_ti_iu_len, ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
DMA_FROM_DEVICE); DMA_FROM_DEVICE);
res = srp_post_recv(target, iu); res = srp_post_recv(ch, iu);
if (res != 0) if (res != 0)
shost_printk(KERN_ERR, target->scsi_host, shost_printk(KERN_ERR, target->scsi_host,
PFX "Recv failed with error code %d\n", res); PFX "Recv failed with error code %d\n", res);
...@@ -1819,33 +1846,35 @@ static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status, ...@@ -1819,33 +1846,35 @@ static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
target->qp_in_error = true; target->qp_in_error = true;
} }
static void srp_recv_completion(struct ib_cq *cq, void *target_ptr) static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr)
{ {
struct srp_target_port *target = target_ptr; struct srp_rdma_ch *ch = ch_ptr;
struct ib_wc wc; struct ib_wc wc;
ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
while (ib_poll_cq(cq, 1, &wc) > 0) { while (ib_poll_cq(cq, 1, &wc) > 0) {
if (likely(wc.status == IB_WC_SUCCESS)) { if (likely(wc.status == IB_WC_SUCCESS)) {
srp_handle_recv(target, &wc); srp_handle_recv(ch, &wc);
} else { } else {
srp_handle_qp_err(wc.wr_id, wc.status, false, target); srp_handle_qp_err(wc.wr_id, wc.status, false,
ch->target);
} }
} }
} }
static void srp_send_completion(struct ib_cq *cq, void *target_ptr) static void srp_send_completion(struct ib_cq *cq, void *ch_ptr)
{ {
struct srp_target_port *target = target_ptr; struct srp_rdma_ch *ch = ch_ptr;
struct ib_wc wc; struct ib_wc wc;
struct srp_iu *iu; struct srp_iu *iu;
while (ib_poll_cq(cq, 1, &wc) > 0) { while (ib_poll_cq(cq, 1, &wc) > 0) {
if (likely(wc.status == IB_WC_SUCCESS)) { if (likely(wc.status == IB_WC_SUCCESS)) {
iu = (struct srp_iu *) (uintptr_t) wc.wr_id; iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
list_add(&iu->list, &target->free_tx); list_add(&iu->list, &ch->free_tx);
} else { } else {
srp_handle_qp_err(wc.wr_id, wc.status, true, target); srp_handle_qp_err(wc.wr_id, wc.status, true,
ch->target);
} }
} }
} }
...@@ -1854,6 +1883,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) ...@@ -1854,6 +1883,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
{ {
struct srp_target_port *target = host_to_target(shost); struct srp_target_port *target = host_to_target(shost);
struct srp_rport *rport = target->rport; struct srp_rport *rport = target->rport;
struct srp_rdma_ch *ch;
struct srp_request *req; struct srp_request *req;
struct srp_iu *iu; struct srp_iu *iu;
struct srp_cmd *cmd; struct srp_cmd *cmd;
...@@ -1875,14 +1905,16 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) ...@@ -1875,14 +1905,16 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
if (unlikely(scmnd->result)) if (unlikely(scmnd->result))
goto err; goto err;
spin_lock_irqsave(&target->lock, flags); ch = &target->ch;
iu = __srp_get_tx_iu(target, SRP_IU_CMD);
spin_lock_irqsave(&ch->lock, flags);
iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
if (!iu) if (!iu)
goto err_unlock; goto err_unlock;
req = list_first_entry(&target->free_reqs, struct srp_request, list); req = list_first_entry(&ch->free_reqs, struct srp_request, list);
list_del(&req->list); list_del(&req->list);
spin_unlock_irqrestore(&target->lock, flags); spin_unlock_irqrestore(&ch->lock, flags);
dev = target->srp_host->srp_dev->dev; dev = target->srp_host->srp_dev->dev;
ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len, ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
...@@ -1901,7 +1933,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) ...@@ -1901,7 +1933,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
req->scmnd = scmnd; req->scmnd = scmnd;
req->cmd = iu; req->cmd = iu;
len = srp_map_data(scmnd, target, req); len = srp_map_data(scmnd, ch, req);
if (len < 0) { if (len < 0) {
shost_printk(KERN_ERR, target->scsi_host, shost_printk(KERN_ERR, target->scsi_host,
PFX "Failed to map data (%d)\n", len); PFX "Failed to map data (%d)\n", len);
...@@ -1919,7 +1951,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) ...@@ -1919,7 +1951,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len, ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
DMA_TO_DEVICE); DMA_TO_DEVICE);
if (srp_post_send(target, iu, len)) { if (srp_post_send(ch, iu, len)) {
shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n"); shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
goto err_unmap; goto err_unmap;
} }
...@@ -1933,10 +1965,10 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) ...@@ -1933,10 +1965,10 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
return ret; return ret;
err_unmap: err_unmap:
srp_unmap_data(scmnd, target, req); srp_unmap_data(scmnd, ch, req);
err_iu: err_iu:
srp_put_tx_iu(target, iu, SRP_IU_CMD); srp_put_tx_iu(ch, iu, SRP_IU_CMD);
/* /*
* Avoid that the loops that iterate over the request ring can * Avoid that the loops that iterate over the request ring can
...@@ -1944,11 +1976,11 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) ...@@ -1944,11 +1976,11 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
*/ */
req->scmnd = NULL; req->scmnd = NULL;
spin_lock_irqsave(&target->lock, flags); spin_lock_irqsave(&ch->lock, flags);
list_add(&req->list, &target->free_reqs); list_add(&req->list, &ch->free_reqs);
err_unlock: err_unlock:
spin_unlock_irqrestore(&target->lock, flags); spin_unlock_irqrestore(&ch->lock, flags);
err: err:
if (scmnd->result) { if (scmnd->result) {
...@@ -1963,53 +1995,54 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) ...@@ -1963,53 +1995,54 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
/* /*
* Note: the resources allocated in this function are freed in * Note: the resources allocated in this function are freed in
* srp_free_target_ib(). * srp_free_ch_ib().
*/ */
static int srp_alloc_iu_bufs(struct srp_target_port *target) static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
int i; int i;
target->rx_ring = kzalloc(target->queue_size * sizeof(*target->rx_ring), ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
GFP_KERNEL); GFP_KERNEL);
if (!target->rx_ring) if (!ch->rx_ring)
goto err_no_ring; goto err_no_ring;
target->tx_ring = kzalloc(target->queue_size * sizeof(*target->tx_ring), ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
GFP_KERNEL); GFP_KERNEL);
if (!target->tx_ring) if (!ch->tx_ring)
goto err_no_ring; goto err_no_ring;
for (i = 0; i < target->queue_size; ++i) { for (i = 0; i < target->queue_size; ++i) {
target->rx_ring[i] = srp_alloc_iu(target->srp_host, ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
target->max_ti_iu_len, ch->max_ti_iu_len,
GFP_KERNEL, DMA_FROM_DEVICE); GFP_KERNEL, DMA_FROM_DEVICE);
if (!target->rx_ring[i]) if (!ch->rx_ring[i])
goto err; goto err;
} }
for (i = 0; i < target->queue_size; ++i) { for (i = 0; i < target->queue_size; ++i) {
target->tx_ring[i] = srp_alloc_iu(target->srp_host, ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
target->max_iu_len, target->max_iu_len,
GFP_KERNEL, DMA_TO_DEVICE); GFP_KERNEL, DMA_TO_DEVICE);
if (!target->tx_ring[i]) if (!ch->tx_ring[i])
goto err; goto err;
list_add(&target->tx_ring[i]->list, &target->free_tx); list_add(&ch->tx_ring[i]->list, &ch->free_tx);
} }
return 0; return 0;
err: err:
for (i = 0; i < target->queue_size; ++i) { for (i = 0; i < target->queue_size; ++i) {
srp_free_iu(target->srp_host, target->rx_ring[i]); srp_free_iu(target->srp_host, ch->rx_ring[i]);
srp_free_iu(target->srp_host, target->tx_ring[i]); srp_free_iu(target->srp_host, ch->tx_ring[i]);
} }
err_no_ring: err_no_ring:
kfree(target->tx_ring); kfree(ch->tx_ring);
target->tx_ring = NULL; ch->tx_ring = NULL;
kfree(target->rx_ring); kfree(ch->rx_ring);
target->rx_ring = NULL; ch->rx_ring = NULL;
return -ENOMEM; return -ENOMEM;
} }
...@@ -2043,23 +2076,24 @@ static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask) ...@@ -2043,23 +2076,24 @@ static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
static void srp_cm_rep_handler(struct ib_cm_id *cm_id, static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
struct srp_login_rsp *lrsp, struct srp_login_rsp *lrsp,
struct srp_target_port *target) struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
struct ib_qp_attr *qp_attr = NULL; struct ib_qp_attr *qp_attr = NULL;
int attr_mask = 0; int attr_mask = 0;
int ret; int ret;
int i; int i;
if (lrsp->opcode == SRP_LOGIN_RSP) { if (lrsp->opcode == SRP_LOGIN_RSP) {
target->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len); ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
target->req_lim = be32_to_cpu(lrsp->req_lim_delta); ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
/* /*
* Reserve credits for task management so we don't * Reserve credits for task management so we don't
* bounce requests back to the SCSI mid-layer. * bounce requests back to the SCSI mid-layer.
*/ */
target->scsi_host->can_queue target->scsi_host->can_queue
= min(target->req_lim - SRP_TSK_MGMT_SQ_SIZE, = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
target->scsi_host->can_queue); target->scsi_host->can_queue);
target->scsi_host->cmd_per_lun target->scsi_host->cmd_per_lun
= min_t(int, target->scsi_host->can_queue, = min_t(int, target->scsi_host->can_queue,
...@@ -2071,8 +2105,8 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id, ...@@ -2071,8 +2105,8 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
goto error; goto error;
} }
if (!target->rx_ring) { if (!ch->rx_ring) {
ret = srp_alloc_iu_bufs(target); ret = srp_alloc_iu_bufs(ch);
if (ret) if (ret)
goto error; goto error;
} }
...@@ -2087,13 +2121,14 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id, ...@@ -2087,13 +2121,14 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
if (ret) if (ret)
goto error_free; goto error_free;
ret = ib_modify_qp(target->qp, qp_attr, attr_mask); ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
if (ret) if (ret)
goto error_free; goto error_free;
for (i = 0; i < target->queue_size; i++) { for (i = 0; i < target->queue_size; i++) {
struct srp_iu *iu = target->rx_ring[i]; struct srp_iu *iu = ch->rx_ring[i];
ret = srp_post_recv(target, iu);
ret = srp_post_recv(ch, iu);
if (ret) if (ret)
goto error_free; goto error_free;
} }
...@@ -2105,7 +2140,7 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id, ...@@ -2105,7 +2140,7 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask); target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
ret = ib_modify_qp(target->qp, qp_attr, attr_mask); ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
if (ret) if (ret)
goto error_free; goto error_free;
...@@ -2115,13 +2150,14 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id, ...@@ -2115,13 +2150,14 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
kfree(qp_attr); kfree(qp_attr);
error: error:
target->status = ret; ch->status = ret;
} }
static void srp_cm_rej_handler(struct ib_cm_id *cm_id, static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
struct ib_cm_event *event, struct ib_cm_event *event,
struct srp_target_port *target) struct srp_rdma_ch *ch)
{ {
struct srp_target_port *target = ch->target;
struct Scsi_Host *shost = target->scsi_host; struct Scsi_Host *shost = target->scsi_host;
struct ib_class_port_info *cpi; struct ib_class_port_info *cpi;
int opcode; int opcode;
...@@ -2129,12 +2165,12 @@ static void srp_cm_rej_handler(struct ib_cm_id *cm_id, ...@@ -2129,12 +2165,12 @@ static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
switch (event->param.rej_rcvd.reason) { switch (event->param.rej_rcvd.reason) {
case IB_CM_REJ_PORT_CM_REDIRECT: case IB_CM_REJ_PORT_CM_REDIRECT:
cpi = event->param.rej_rcvd.ari; cpi = event->param.rej_rcvd.ari;
target->path.dlid = cpi->redirect_lid; ch->path.dlid = cpi->redirect_lid;
target->path.pkey = cpi->redirect_pkey; ch->path.pkey = cpi->redirect_pkey;
cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff; cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
memcpy(target->path.dgid.raw, cpi->redirect_gid, 16); memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
target->status = target->path.dlid ? ch->status = ch->path.dlid ?
SRP_DLID_REDIRECT : SRP_PORT_REDIRECT; SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
break; break;
...@@ -2145,26 +2181,26 @@ static void srp_cm_rej_handler(struct ib_cm_id *cm_id, ...@@ -2145,26 +2181,26 @@ static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
* reject reason code 25 when they mean 24 * reject reason code 25 when they mean 24
* (port redirect). * (port redirect).
*/ */
memcpy(target->path.dgid.raw, memcpy(ch->path.dgid.raw,
event->param.rej_rcvd.ari, 16); event->param.rej_rcvd.ari, 16);
shost_printk(KERN_DEBUG, shost, shost_printk(KERN_DEBUG, shost,
PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n", PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
(unsigned long long) be64_to_cpu(target->path.dgid.global.subnet_prefix), be64_to_cpu(ch->path.dgid.global.subnet_prefix),
(unsigned long long) be64_to_cpu(target->path.dgid.global.interface_id)); be64_to_cpu(ch->path.dgid.global.interface_id));
target->status = SRP_PORT_REDIRECT; ch->status = SRP_PORT_REDIRECT;
} else { } else {
shost_printk(KERN_WARNING, shost, shost_printk(KERN_WARNING, shost,
" REJ reason: IB_CM_REJ_PORT_REDIRECT\n"); " REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
target->status = -ECONNRESET; ch->status = -ECONNRESET;
} }
break; break;
case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
shost_printk(KERN_WARNING, shost, shost_printk(KERN_WARNING, shost,
" REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
target->status = -ECONNRESET; ch->status = -ECONNRESET;
break; break;
case IB_CM_REJ_CONSUMER_DEFINED: case IB_CM_REJ_CONSUMER_DEFINED:
...@@ -2185,24 +2221,25 @@ static void srp_cm_rej_handler(struct ib_cm_id *cm_id, ...@@ -2185,24 +2221,25 @@ static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
shost_printk(KERN_WARNING, shost, shost_printk(KERN_WARNING, shost,
" REJ reason: IB_CM_REJ_CONSUMER_DEFINED," " REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
" opcode 0x%02x\n", opcode); " opcode 0x%02x\n", opcode);
target->status = -ECONNRESET; ch->status = -ECONNRESET;
break; break;
case IB_CM_REJ_STALE_CONN: case IB_CM_REJ_STALE_CONN:
shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n"); shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n");
target->status = SRP_STALE_CONN; ch->status = SRP_STALE_CONN;
break; break;
default: default:
shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n",
event->param.rej_rcvd.reason); event->param.rej_rcvd.reason);
target->status = -ECONNRESET; ch->status = -ECONNRESET;
} }
} }
static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event) static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
{ {
struct srp_target_port *target = cm_id->context; struct srp_rdma_ch *ch = cm_id->context;
struct srp_target_port *target = ch->target;
int comp = 0; int comp = 0;
switch (event->event) { switch (event->event) {
...@@ -2210,19 +2247,19 @@ static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event) ...@@ -2210,19 +2247,19 @@ static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
shost_printk(KERN_DEBUG, target->scsi_host, shost_printk(KERN_DEBUG, target->scsi_host,
PFX "Sending CM REQ failed\n"); PFX "Sending CM REQ failed\n");
comp = 1; comp = 1;
target->status = -ECONNRESET; ch->status = -ECONNRESET;
break; break;
case IB_CM_REP_RECEIVED: case IB_CM_REP_RECEIVED:
comp = 1; comp = 1;
srp_cm_rep_handler(cm_id, event->private_data, target); srp_cm_rep_handler(cm_id, event->private_data, ch);
break; break;
case IB_CM_REJ_RECEIVED: case IB_CM_REJ_RECEIVED:
shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
comp = 1; comp = 1;
srp_cm_rej_handler(cm_id, event, target); srp_cm_rej_handler(cm_id, event, ch);
break; break;
case IB_CM_DREQ_RECEIVED: case IB_CM_DREQ_RECEIVED:
...@@ -2240,7 +2277,7 @@ static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event) ...@@ -2240,7 +2277,7 @@ static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
PFX "connection closed\n"); PFX "connection closed\n");
comp = 1; comp = 1;
target->status = 0; ch->status = 0;
break; break;
case IB_CM_MRA_RECEIVED: case IB_CM_MRA_RECEIVED:
...@@ -2255,7 +2292,7 @@ static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event) ...@@ -2255,7 +2292,7 @@ static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
} }
if (comp) if (comp)
complete(&target->done); complete(&ch->done);
return 0; return 0;
} }
...@@ -2289,9 +2326,10 @@ srp_change_queue_depth(struct scsi_device *sdev, int qdepth, int reason) ...@@ -2289,9 +2326,10 @@ srp_change_queue_depth(struct scsi_device *sdev, int qdepth, int reason)
return sdev->queue_depth; return sdev->queue_depth;
} }
static int srp_send_tsk_mgmt(struct srp_target_port *target, static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag,
u64 req_tag, unsigned int lun, u8 func) unsigned int lun, u8 func)
{ {
struct srp_target_port *target = ch->target;
struct srp_rport *rport = target->rport; struct srp_rport *rport = target->rport;
struct ib_device *dev = target->srp_host->srp_dev->dev; struct ib_device *dev = target->srp_host->srp_dev->dev;
struct srp_iu *iu; struct srp_iu *iu;
...@@ -2300,16 +2338,16 @@ static int srp_send_tsk_mgmt(struct srp_target_port *target, ...@@ -2300,16 +2338,16 @@ static int srp_send_tsk_mgmt(struct srp_target_port *target,
if (!target->connected || target->qp_in_error) if (!target->connected || target->qp_in_error)
return -1; return -1;
init_completion(&target->tsk_mgmt_done); init_completion(&ch->tsk_mgmt_done);
/* /*
* Lock the rport mutex to avoid that srp_create_target_ib() is * Lock the rport mutex to avoid that srp_create_ch_ib() is
* invoked while a task management function is being sent. * invoked while a task management function is being sent.
*/ */
mutex_lock(&rport->mutex); mutex_lock(&rport->mutex);
spin_lock_irq(&target->lock); spin_lock_irq(&ch->lock);
iu = __srp_get_tx_iu(target, SRP_IU_TSK_MGMT); iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
spin_unlock_irq(&target->lock); spin_unlock_irq(&ch->lock);
if (!iu) { if (!iu) {
mutex_unlock(&rport->mutex); mutex_unlock(&rport->mutex);
...@@ -2330,15 +2368,15 @@ static int srp_send_tsk_mgmt(struct srp_target_port *target, ...@@ -2330,15 +2368,15 @@ static int srp_send_tsk_mgmt(struct srp_target_port *target,
ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt, ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
DMA_TO_DEVICE); DMA_TO_DEVICE);
if (srp_post_send(target, iu, sizeof *tsk_mgmt)) { if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
srp_put_tx_iu(target, iu, SRP_IU_TSK_MGMT); srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
mutex_unlock(&rport->mutex); mutex_unlock(&rport->mutex);
return -1; return -1;
} }
mutex_unlock(&rport->mutex); mutex_unlock(&rport->mutex);
if (!wait_for_completion_timeout(&target->tsk_mgmt_done, if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS))) msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
return -1; return -1;
...@@ -2349,20 +2387,22 @@ static int srp_abort(struct scsi_cmnd *scmnd) ...@@ -2349,20 +2387,22 @@ static int srp_abort(struct scsi_cmnd *scmnd)
{ {
struct srp_target_port *target = host_to_target(scmnd->device->host); struct srp_target_port *target = host_to_target(scmnd->device->host);
struct srp_request *req = (struct srp_request *) scmnd->host_scribble; struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
struct srp_rdma_ch *ch;
int ret; int ret;
shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n"); shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
if (!req || !srp_claim_req(target, req, NULL, scmnd)) ch = &target->ch;
if (!req || !srp_claim_req(ch, req, NULL, scmnd))
return SUCCESS; return SUCCESS;
if (srp_send_tsk_mgmt(target, req->index, scmnd->device->lun, if (srp_send_tsk_mgmt(ch, req->index, scmnd->device->lun,
SRP_TSK_ABORT_TASK) == 0) SRP_TSK_ABORT_TASK) == 0)
ret = SUCCESS; ret = SUCCESS;
else if (target->rport->state == SRP_RPORT_LOST) else if (target->rport->state == SRP_RPORT_LOST)
ret = FAST_IO_FAIL; ret = FAST_IO_FAIL;
else else
ret = FAILED; ret = FAILED;
srp_free_req(target, req, scmnd, 0); srp_free_req(ch, req, scmnd, 0);
scmnd->result = DID_ABORT << 16; scmnd->result = DID_ABORT << 16;
scmnd->scsi_done(scmnd); scmnd->scsi_done(scmnd);
...@@ -2372,19 +2412,21 @@ static int srp_abort(struct scsi_cmnd *scmnd) ...@@ -2372,19 +2412,21 @@ static int srp_abort(struct scsi_cmnd *scmnd)
static int srp_reset_device(struct scsi_cmnd *scmnd) static int srp_reset_device(struct scsi_cmnd *scmnd)
{ {
struct srp_target_port *target = host_to_target(scmnd->device->host); struct srp_target_port *target = host_to_target(scmnd->device->host);
struct srp_rdma_ch *ch = &target->ch;
int i; int i;
shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n"); shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
if (srp_send_tsk_mgmt(target, SRP_TAG_NO_REQ, scmnd->device->lun, if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
SRP_TSK_LUN_RESET)) SRP_TSK_LUN_RESET))
return FAILED; return FAILED;
if (target->tsk_mgmt_status) if (ch->tsk_mgmt_status)
return FAILED; return FAILED;
for (i = 0; i < target->req_ring_size; ++i) { for (i = 0; i < target->req_ring_size; ++i) {
struct srp_request *req = &target->req_ring[i]; struct srp_request *req = &ch->req_ring[i];
srp_finish_req(target, req, scmnd->device, DID_RESET << 16);
srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
} }
return SUCCESS; return SUCCESS;
...@@ -2461,8 +2503,9 @@ static ssize_t show_dgid(struct device *dev, struct device_attribute *attr, ...@@ -2461,8 +2503,9 @@ static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
char *buf) char *buf)
{ {
struct srp_target_port *target = host_to_target(class_to_shost(dev)); struct srp_target_port *target = host_to_target(class_to_shost(dev));
struct srp_rdma_ch *ch = &target->ch;
return sprintf(buf, "%pI6\n", target->path.dgid.raw); return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
} }
static ssize_t show_orig_dgid(struct device *dev, static ssize_t show_orig_dgid(struct device *dev,
...@@ -2478,7 +2521,7 @@ static ssize_t show_req_lim(struct device *dev, ...@@ -2478,7 +2521,7 @@ static ssize_t show_req_lim(struct device *dev,
{ {
struct srp_target_port *target = host_to_target(class_to_shost(dev)); struct srp_target_port *target = host_to_target(class_to_shost(dev));
return sprintf(buf, "%d\n", target->req_lim); return sprintf(buf, "%d\n", target->ch.req_lim);
} }
static ssize_t show_zero_req_lim(struct device *dev, static ssize_t show_zero_req_lim(struct device *dev,
...@@ -2970,6 +3013,7 @@ static ssize_t srp_create_target(struct device *dev, ...@@ -2970,6 +3013,7 @@ static ssize_t srp_create_target(struct device *dev,
container_of(dev, struct srp_host, dev); container_of(dev, struct srp_host, dev);
struct Scsi_Host *target_host; struct Scsi_Host *target_host;
struct srp_target_port *target; struct srp_target_port *target;
struct srp_rdma_ch *ch;
struct srp_device *srp_dev = host->srp_dev; struct srp_device *srp_dev = host->srp_dev;
struct ib_device *ibdev = srp_dev->dev; struct ib_device *ibdev = srp_dev->dev;
int ret; int ret;
...@@ -3038,8 +3082,12 @@ static ssize_t srp_create_target(struct device *dev, ...@@ -3038,8 +3082,12 @@ static ssize_t srp_create_target(struct device *dev,
INIT_WORK(&target->tl_err_work, srp_tl_err_work); INIT_WORK(&target->tl_err_work, srp_tl_err_work);
INIT_WORK(&target->remove_work, srp_remove_work); INIT_WORK(&target->remove_work, srp_remove_work);
spin_lock_init(&target->lock); spin_lock_init(&target->lock);
INIT_LIST_HEAD(&target->free_tx); ch = &target->ch;
ret = srp_alloc_req_data(target); ch->target = target;
ch->comp_vector = target->comp_vector;
spin_lock_init(&ch->lock);
INIT_LIST_HEAD(&ch->free_tx);
ret = srp_alloc_req_data(ch);
if (ret) if (ret)
goto err_free_mem; goto err_free_mem;
...@@ -3047,15 +3095,15 @@ static ssize_t srp_create_target(struct device *dev, ...@@ -3047,15 +3095,15 @@ static ssize_t srp_create_target(struct device *dev,
if (ret) if (ret)
goto err_free_mem; goto err_free_mem;
ret = srp_create_target_ib(target); ret = srp_create_ch_ib(ch);
if (ret) if (ret)
goto err_free_mem; goto err_free_mem;
ret = srp_new_cm_id(target); ret = srp_new_cm_id(ch);
if (ret) if (ret)
goto err_free_ib; goto err_free_ib;
ret = srp_connect_target(target); ret = srp_connect_ch(ch);
if (ret) { if (ret) {
shost_printk(KERN_ERR, target->scsi_host, shost_printk(KERN_ERR, target->scsi_host,
PFX "Connection failed\n"); PFX "Connection failed\n");
...@@ -3089,10 +3137,10 @@ static ssize_t srp_create_target(struct device *dev, ...@@ -3089,10 +3137,10 @@ static ssize_t srp_create_target(struct device *dev,
srp_disconnect_target(target); srp_disconnect_target(target);
err_free_ib: err_free_ib:
srp_free_target_ib(target); srp_free_ch_ib(target, ch);
err_free_mem: err_free_mem:
srp_free_req_data(target); srp_free_req_data(target, ch);
err: err:
scsi_host_put(target_host); scsi_host_put(target_host);
......
...@@ -130,7 +130,11 @@ struct srp_request { ...@@ -130,7 +130,11 @@ struct srp_request {
short index; short index;
}; };
struct srp_target_port { /**
* struct srp_rdma_ch
* @comp_vector: Completion vector used by this RDMA channel.
*/
struct srp_rdma_ch {
/* These are RW in the hot path, and commonly used together */ /* These are RW in the hot path, and commonly used together */
struct list_head free_tx; struct list_head free_tx;
struct list_head free_reqs; struct list_head free_reqs;
...@@ -138,13 +142,48 @@ struct srp_target_port { ...@@ -138,13 +142,48 @@ struct srp_target_port {
s32 req_lim; s32 req_lim;
/* These are read-only in the hot path */ /* These are read-only in the hot path */
struct ib_cq *send_cq ____cacheline_aligned_in_smp; struct srp_target_port *target ____cacheline_aligned_in_smp;
struct ib_cq *send_cq;
struct ib_cq *recv_cq; struct ib_cq *recv_cq;
struct ib_qp *qp; struct ib_qp *qp;
union { union {
struct ib_fmr_pool *fmr_pool; struct ib_fmr_pool *fmr_pool;
struct srp_fr_pool *fr_pool; struct srp_fr_pool *fr_pool;
}; };
/* Everything above this point is used in the hot path of
* command processing. Try to keep them packed into cachelines.
*/
struct completion done;
int status;
struct ib_sa_path_rec path;
struct ib_sa_query *path_query;
int path_query_id;
struct ib_cm_id *cm_id;
struct srp_iu **tx_ring;
struct srp_iu **rx_ring;
struct srp_request *req_ring;
int max_ti_iu_len;
int comp_vector;
struct completion tsk_mgmt_done;
u8 tsk_mgmt_status;
};
/**
* struct srp_target_port
* @comp_vector: Completion vector used by the first RDMA channel created for
* this target port.
*/
struct srp_target_port {
/* read and written in the hot path */
spinlock_t lock;
struct srp_rdma_ch ch;
/* read only in the hot path */
u32 lkey; u32 lkey;
u32 rkey; u32 rkey;
enum srp_target_state state; enum srp_target_state state;
...@@ -153,10 +192,7 @@ struct srp_target_port { ...@@ -153,10 +192,7 @@ struct srp_target_port {
unsigned int indirect_size; unsigned int indirect_size;
bool allow_ext_sg; bool allow_ext_sg;
/* Everything above this point is used in the hot path of /* other member variables */
* command processing. Try to keep them packed into cachelines.
*/
union ib_gid sgid; union ib_gid sgid;
__be64 id_ext; __be64 id_ext;
__be64 ioc_guid; __be64 ioc_guid;
...@@ -176,33 +212,17 @@ struct srp_target_port { ...@@ -176,33 +212,17 @@ struct srp_target_port {
union ib_gid orig_dgid; union ib_gid orig_dgid;
__be16 pkey; __be16 pkey;
struct ib_sa_path_rec path;
struct ib_sa_query *path_query;
int path_query_id;
u32 rq_tmo_jiffies; u32 rq_tmo_jiffies;
bool connected; bool connected;
struct ib_cm_id *cm_id;
int max_ti_iu_len;
int zero_req_lim; int zero_req_lim;
struct srp_iu **tx_ring;
struct srp_iu **rx_ring;
struct srp_request *req_ring;
struct work_struct tl_err_work; struct work_struct tl_err_work;
struct work_struct remove_work; struct work_struct remove_work;
struct list_head list; struct list_head list;
struct completion done;
int status;
bool qp_in_error; bool qp_in_error;
struct completion tsk_mgmt_done;
u8 tsk_mgmt_status;
}; };
struct srp_iu { struct srp_iu {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment