Commit f8a05a1d authored by Jens Axboe's avatar Jens Axboe

Merge branch 'nvme-4.12' of git://git.infradead.org/nvme into for-4.12/block

Christoph writes:

This is the current NVMe pile: virtualization extensions, lots of FC
updates and various misc bits.  There are a few more FC bits that didn't
make the cut, but we'd like to get this request out before the merge
window for sure.
parents 95c55ff4 e02ab023
...@@ -75,6 +75,20 @@ static int nvme_error_status(struct request *req) ...@@ -75,6 +75,20 @@ static int nvme_error_status(struct request *req)
return -ENOSPC; return -ENOSPC;
default: default:
return -EIO; return -EIO;
/*
* XXX: these errors are a nasty side-band protocol to
* drivers/md/dm-mpath.c:noretry_error() that aren't documented
* anywhere..
*/
case NVME_SC_CMD_SEQ_ERROR:
return -EILSEQ;
case NVME_SC_ONCS_NOT_SUPPORTED:
return -EOPNOTSUPP;
case NVME_SC_WRITE_FAULT:
case NVME_SC_READ_ERROR:
case NVME_SC_UNWRITTEN_BLOCK:
return -ENODATA;
} }
} }
......
...@@ -61,16 +61,23 @@ struct nvme_fc_queue { ...@@ -61,16 +61,23 @@ struct nvme_fc_queue {
unsigned long flags; unsigned long flags;
} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
enum nvme_fcop_flags {
FCOP_FLAGS_TERMIO = (1 << 0),
FCOP_FLAGS_RELEASED = (1 << 1),
FCOP_FLAGS_COMPLETE = (1 << 2),
};
struct nvmefc_ls_req_op { struct nvmefc_ls_req_op {
struct nvmefc_ls_req ls_req; struct nvmefc_ls_req ls_req;
struct nvme_fc_ctrl *ctrl; struct nvme_fc_rport *rport;
struct nvme_fc_queue *queue; struct nvme_fc_queue *queue;
struct request *rq; struct request *rq;
u32 flags;
int ls_error; int ls_error;
struct completion ls_done; struct completion ls_done;
struct list_head lsreq_list; /* ctrl->ls_req_list */ struct list_head lsreq_list; /* rport->ls_req_list */
bool req_queued; bool req_queued;
}; };
...@@ -120,6 +127,9 @@ struct nvme_fc_rport { ...@@ -120,6 +127,9 @@ struct nvme_fc_rport {
struct list_head endp_list; /* for lport->endp_list */ struct list_head endp_list; /* for lport->endp_list */
struct list_head ctrl_list; struct list_head ctrl_list;
struct list_head ls_req_list;
struct device *dev; /* physical device for dma */
struct nvme_fc_lport *lport;
spinlock_t lock; spinlock_t lock;
struct kref ref; struct kref ref;
} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
...@@ -144,7 +154,6 @@ struct nvme_fc_ctrl { ...@@ -144,7 +154,6 @@ struct nvme_fc_ctrl {
u64 cap; u64 cap;
struct list_head ctrl_list; /* rport->ctrl_list */ struct list_head ctrl_list; /* rport->ctrl_list */
struct list_head ls_req_list;
struct blk_mq_tag_set admin_tag_set; struct blk_mq_tag_set admin_tag_set;
struct blk_mq_tag_set tag_set; struct blk_mq_tag_set tag_set;
...@@ -419,9 +428,12 @@ nvme_fc_register_remoteport(struct nvme_fc_local_port *localport, ...@@ -419,9 +428,12 @@ nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
INIT_LIST_HEAD(&newrec->endp_list); INIT_LIST_HEAD(&newrec->endp_list);
INIT_LIST_HEAD(&newrec->ctrl_list); INIT_LIST_HEAD(&newrec->ctrl_list);
INIT_LIST_HEAD(&newrec->ls_req_list);
kref_init(&newrec->ref); kref_init(&newrec->ref);
spin_lock_init(&newrec->lock); spin_lock_init(&newrec->lock);
newrec->remoteport.localport = &lport->localport; newrec->remoteport.localport = &lport->localport;
newrec->dev = lport->dev;
newrec->lport = lport;
newrec->remoteport.private = &newrec[1]; newrec->remoteport.private = &newrec[1];
newrec->remoteport.port_role = pinfo->port_role; newrec->remoteport.port_role = pinfo->port_role;
newrec->remoteport.node_name = pinfo->node_name; newrec->remoteport.node_name = pinfo->node_name;
...@@ -444,7 +456,6 @@ nvme_fc_register_remoteport(struct nvme_fc_local_port *localport, ...@@ -444,7 +456,6 @@ nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
out_reghost_failed: out_reghost_failed:
*portptr = NULL; *portptr = NULL;
return ret; return ret;
} }
EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport); EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
...@@ -487,6 +498,30 @@ nvme_fc_rport_get(struct nvme_fc_rport *rport) ...@@ -487,6 +498,30 @@ nvme_fc_rport_get(struct nvme_fc_rport *rport)
return kref_get_unless_zero(&rport->ref); return kref_get_unless_zero(&rport->ref);
} }
static int
nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
{
struct nvmefc_ls_req_op *lsop;
unsigned long flags;
restart:
spin_lock_irqsave(&rport->lock, flags);
list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
lsop->flags |= FCOP_FLAGS_TERMIO;
spin_unlock_irqrestore(&rport->lock, flags);
rport->lport->ops->ls_abort(&rport->lport->localport,
&rport->remoteport,
&lsop->ls_req);
goto restart;
}
}
spin_unlock_irqrestore(&rport->lock, flags);
return 0;
}
/** /**
* nvme_fc_unregister_remoteport - transport entry point called by an * nvme_fc_unregister_remoteport - transport entry point called by an
* LLDD to deregister/remove a previously * LLDD to deregister/remove a previously
...@@ -522,6 +557,8 @@ nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr) ...@@ -522,6 +557,8 @@ nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
spin_unlock_irqrestore(&rport->lock, flags); spin_unlock_irqrestore(&rport->lock, flags);
nvme_fc_abort_lsops(rport);
nvme_fc_rport_put(rport); nvme_fc_rport_put(rport);
return 0; return 0;
} }
...@@ -624,16 +661,16 @@ static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *); ...@@ -624,16 +661,16 @@ static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
static void static void
__nvme_fc_finish_ls_req(struct nvme_fc_ctrl *ctrl, __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
struct nvmefc_ls_req_op *lsop)
{ {
struct nvme_fc_rport *rport = lsop->rport;
struct nvmefc_ls_req *lsreq = &lsop->ls_req; struct nvmefc_ls_req *lsreq = &lsop->ls_req;
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&ctrl->lock, flags); spin_lock_irqsave(&rport->lock, flags);
if (!lsop->req_queued) { if (!lsop->req_queued) {
spin_unlock_irqrestore(&ctrl->lock, flags); spin_unlock_irqrestore(&rport->lock, flags);
return; return;
} }
...@@ -641,56 +678,71 @@ __nvme_fc_finish_ls_req(struct nvme_fc_ctrl *ctrl, ...@@ -641,56 +678,71 @@ __nvme_fc_finish_ls_req(struct nvme_fc_ctrl *ctrl,
lsop->req_queued = false; lsop->req_queued = false;
spin_unlock_irqrestore(&ctrl->lock, flags); spin_unlock_irqrestore(&rport->lock, flags);
fc_dma_unmap_single(ctrl->dev, lsreq->rqstdma, fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
(lsreq->rqstlen + lsreq->rsplen), (lsreq->rqstlen + lsreq->rsplen),
DMA_BIDIRECTIONAL); DMA_BIDIRECTIONAL);
nvme_fc_ctrl_put(ctrl); nvme_fc_rport_put(rport);
} }
static int static int
__nvme_fc_send_ls_req(struct nvme_fc_ctrl *ctrl, __nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
struct nvmefc_ls_req_op *lsop, struct nvmefc_ls_req_op *lsop,
void (*done)(struct nvmefc_ls_req *req, int status)) void (*done)(struct nvmefc_ls_req *req, int status))
{ {
struct nvmefc_ls_req *lsreq = &lsop->ls_req; struct nvmefc_ls_req *lsreq = &lsop->ls_req;
unsigned long flags; unsigned long flags;
int ret; int ret = 0;
if (!nvme_fc_ctrl_get(ctrl)) if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
return -ECONNREFUSED;
if (!nvme_fc_rport_get(rport))
return -ESHUTDOWN; return -ESHUTDOWN;
lsreq->done = done; lsreq->done = done;
lsop->ctrl = ctrl; lsop->rport = rport;
lsop->req_queued = false; lsop->req_queued = false;
INIT_LIST_HEAD(&lsop->lsreq_list); INIT_LIST_HEAD(&lsop->lsreq_list);
init_completion(&lsop->ls_done); init_completion(&lsop->ls_done);
lsreq->rqstdma = fc_dma_map_single(ctrl->dev, lsreq->rqstaddr, lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
lsreq->rqstlen + lsreq->rsplen, lsreq->rqstlen + lsreq->rsplen,
DMA_BIDIRECTIONAL); DMA_BIDIRECTIONAL);
if (fc_dma_mapping_error(ctrl->dev, lsreq->rqstdma)) { if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
nvme_fc_ctrl_put(ctrl); ret = -EFAULT;
dev_err(ctrl->dev, goto out_putrport;
"els request command failed EFAULT.\n");
return -EFAULT;
} }
lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen; lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
spin_lock_irqsave(&ctrl->lock, flags); spin_lock_irqsave(&rport->lock, flags);
list_add_tail(&lsop->lsreq_list, &ctrl->ls_req_list); list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
lsop->req_queued = true; lsop->req_queued = true;
spin_unlock_irqrestore(&ctrl->lock, flags); spin_unlock_irqrestore(&rport->lock, flags);
ret = ctrl->lport->ops->ls_req(&ctrl->lport->localport, ret = rport->lport->ops->ls_req(&rport->lport->localport,
&ctrl->rport->remoteport, lsreq); &rport->remoteport, lsreq);
if (ret) if (ret)
goto out_unlink;
return 0;
out_unlink:
lsop->ls_error = ret; lsop->ls_error = ret;
spin_lock_irqsave(&rport->lock, flags);
lsop->req_queued = false;
list_del(&lsop->lsreq_list);
spin_unlock_irqrestore(&rport->lock, flags);
fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
(lsreq->rqstlen + lsreq->rsplen),
DMA_BIDIRECTIONAL);
out_putrport:
nvme_fc_rport_put(rport);
return ret; return ret;
} }
...@@ -705,15 +757,15 @@ nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status) ...@@ -705,15 +757,15 @@ nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
} }
static int static int
nvme_fc_send_ls_req(struct nvme_fc_ctrl *ctrl, struct nvmefc_ls_req_op *lsop) nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
{ {
struct nvmefc_ls_req *lsreq = &lsop->ls_req; struct nvmefc_ls_req *lsreq = &lsop->ls_req;
struct fcnvme_ls_rjt *rjt = lsreq->rspaddr; struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
int ret; int ret;
ret = __nvme_fc_send_ls_req(ctrl, lsop, nvme_fc_send_ls_req_done); ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
if (!ret) if (!ret) {
/* /*
* No timeout/not interruptible as we need the struct * No timeout/not interruptible as we need the struct
* to exist until the lldd calls us back. Thus mandate * to exist until the lldd calls us back. Thus mandate
...@@ -722,14 +774,14 @@ nvme_fc_send_ls_req(struct nvme_fc_ctrl *ctrl, struct nvmefc_ls_req_op *lsop) ...@@ -722,14 +774,14 @@ nvme_fc_send_ls_req(struct nvme_fc_ctrl *ctrl, struct nvmefc_ls_req_op *lsop)
*/ */
wait_for_completion(&lsop->ls_done); wait_for_completion(&lsop->ls_done);
__nvme_fc_finish_ls_req(ctrl, lsop); __nvme_fc_finish_ls_req(lsop);
if (ret) { ret = lsop->ls_error;
dev_err(ctrl->dev,
"ls request command failed (%d).\n", ret);
return ret;
} }
if (ret)
return ret;
/* ACC or RJT payload ? */ /* ACC or RJT payload ? */
if (rjt->w0.ls_cmd == FCNVME_LS_RJT) if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
return -ENXIO; return -ENXIO;
...@@ -737,19 +789,14 @@ nvme_fc_send_ls_req(struct nvme_fc_ctrl *ctrl, struct nvmefc_ls_req_op *lsop) ...@@ -737,19 +789,14 @@ nvme_fc_send_ls_req(struct nvme_fc_ctrl *ctrl, struct nvmefc_ls_req_op *lsop)
return 0; return 0;
} }
static void static int
nvme_fc_send_ls_req_async(struct nvme_fc_ctrl *ctrl, nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
struct nvmefc_ls_req_op *lsop, struct nvmefc_ls_req_op *lsop,
void (*done)(struct nvmefc_ls_req *req, int status)) void (*done)(struct nvmefc_ls_req *req, int status))
{ {
int ret;
ret = __nvme_fc_send_ls_req(ctrl, lsop, done);
/* don't wait for completion */ /* don't wait for completion */
if (ret) return __nvme_fc_send_ls_req(rport, lsop, done);
done(&lsop->ls_req, ret);
} }
/* Validation Error indexes into the string table below */ /* Validation Error indexes into the string table below */
...@@ -839,7 +886,7 @@ nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl, ...@@ -839,7 +886,7 @@ nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
lsreq->rsplen = sizeof(*assoc_acc); lsreq->rsplen = sizeof(*assoc_acc);
lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC; lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;
ret = nvme_fc_send_ls_req(ctrl, lsop); ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
if (ret) if (ret)
goto out_free_buffer; goto out_free_buffer;
...@@ -947,7 +994,7 @@ nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, ...@@ -947,7 +994,7 @@ nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
lsreq->rsplen = sizeof(*conn_acc); lsreq->rsplen = sizeof(*conn_acc);
lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC; lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;
ret = nvme_fc_send_ls_req(ctrl, lsop); ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
if (ret) if (ret)
goto out_free_buffer; goto out_free_buffer;
...@@ -998,14 +1045,8 @@ static void ...@@ -998,14 +1045,8 @@ static void
nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
{ {
struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq); struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
struct nvme_fc_ctrl *ctrl = lsop->ctrl;
__nvme_fc_finish_ls_req(ctrl, lsop); __nvme_fc_finish_ls_req(lsop);
if (status)
dev_err(ctrl->dev,
"disconnect assoc ls request command failed (%d).\n",
status);
/* fc-nvme iniator doesn't care about success or failure of cmd */ /* fc-nvme iniator doesn't care about success or failure of cmd */
...@@ -1036,6 +1077,7 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) ...@@ -1036,6 +1077,7 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
struct fcnvme_ls_disconnect_acc *discon_acc; struct fcnvme_ls_disconnect_acc *discon_acc;
struct nvmefc_ls_req_op *lsop; struct nvmefc_ls_req_op *lsop;
struct nvmefc_ls_req *lsreq; struct nvmefc_ls_req *lsreq;
int ret;
lsop = kzalloc((sizeof(*lsop) + lsop = kzalloc((sizeof(*lsop) +
ctrl->lport->ops->lsrqst_priv_sz + ctrl->lport->ops->lsrqst_priv_sz +
...@@ -1078,7 +1120,10 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) ...@@ -1078,7 +1120,10 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
lsreq->rsplen = sizeof(*discon_acc); lsreq->rsplen = sizeof(*discon_acc);
lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC; lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;
nvme_fc_send_ls_req_async(ctrl, lsop, nvme_fc_disconnect_assoc_done); ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
nvme_fc_disconnect_assoc_done);
if (ret)
kfree(lsop);
/* only meaningful part to terminating the association */ /* only meaningful part to terminating the association */
ctrl->association_id = 0; ctrl->association_id = 0;
...@@ -2302,7 +2347,6 @@ __nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, ...@@ -2302,7 +2347,6 @@ __nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
ctrl->ctrl.opts = opts; ctrl->ctrl.opts = opts;
INIT_LIST_HEAD(&ctrl->ctrl_list); INIT_LIST_HEAD(&ctrl->ctrl_list);
INIT_LIST_HEAD(&ctrl->ls_req_list);
ctrl->lport = lport; ctrl->lport = lport;
ctrl->rport = rport; ctrl->rport = rport;
ctrl->dev = lport->dev; ctrl->dev = lport->dev;
......
...@@ -103,8 +103,22 @@ struct nvme_dev { ...@@ -103,8 +103,22 @@ struct nvme_dev {
u32 cmbloc; u32 cmbloc;
struct nvme_ctrl ctrl; struct nvme_ctrl ctrl;
struct completion ioq_wait; struct completion ioq_wait;
u32 *dbbuf_dbs;
dma_addr_t dbbuf_dbs_dma_addr;
u32 *dbbuf_eis;
dma_addr_t dbbuf_eis_dma_addr;
}; };
static inline unsigned int sq_idx(unsigned int qid, u32 stride)
{
return qid * 2 * stride;
}
static inline unsigned int cq_idx(unsigned int qid, u32 stride)
{
return (qid * 2 + 1) * stride;
}
static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl) static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
{ {
return container_of(ctrl, struct nvme_dev, ctrl); return container_of(ctrl, struct nvme_dev, ctrl);
...@@ -133,6 +147,10 @@ struct nvme_queue { ...@@ -133,6 +147,10 @@ struct nvme_queue {
u16 qid; u16 qid;
u8 cq_phase; u8 cq_phase;
u8 cqe_seen; u8 cqe_seen;
u32 *dbbuf_sq_db;
u32 *dbbuf_cq_db;
u32 *dbbuf_sq_ei;
u32 *dbbuf_cq_ei;
}; };
/* /*
...@@ -171,6 +189,112 @@ static inline void _nvme_check_size(void) ...@@ -171,6 +189,112 @@ static inline void _nvme_check_size(void)
BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
}
static inline unsigned int nvme_dbbuf_size(u32 stride)
{
return ((num_possible_cpus() + 1) * 8 * stride);
}
static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
{
unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);
if (dev->dbbuf_dbs)
return 0;
dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
&dev->dbbuf_dbs_dma_addr,
GFP_KERNEL);
if (!dev->dbbuf_dbs)
return -ENOMEM;
dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
&dev->dbbuf_eis_dma_addr,
GFP_KERNEL);
if (!dev->dbbuf_eis) {
dma_free_coherent(dev->dev, mem_size,
dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
dev->dbbuf_dbs = NULL;
return -ENOMEM;
}
return 0;
}
static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
{
unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);
if (dev->dbbuf_dbs) {
dma_free_coherent(dev->dev, mem_size,
dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
dev->dbbuf_dbs = NULL;
}
if (dev->dbbuf_eis) {
dma_free_coherent(dev->dev, mem_size,
dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
dev->dbbuf_eis = NULL;
}
}
static void nvme_dbbuf_init(struct nvme_dev *dev,
struct nvme_queue *nvmeq, int qid)
{
if (!dev->dbbuf_dbs || !qid)
return;
nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
}
static void nvme_dbbuf_set(struct nvme_dev *dev)
{
struct nvme_command c;
if (!dev->dbbuf_dbs)
return;
memset(&c, 0, sizeof(c));
c.dbbuf.opcode = nvme_admin_dbbuf;
c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);
if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
dev_warn(dev->dev, "unable to set dbbuf\n");
/* Free memory and continue on */
nvme_dbbuf_dma_free(dev);
}
}
static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
{
return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
}
/* Update dbbuf and return true if an MMIO is required */
static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db,
volatile u32 *dbbuf_ei)
{
if (dbbuf_db) {
u16 old_value;
/*
* Ensure that the queue is written before updating
* the doorbell in memory
*/
wmb();
old_value = *dbbuf_db;
*dbbuf_db = value;
if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value))
return false;
}
return true;
} }
/* /*
...@@ -297,6 +421,8 @@ static void __nvme_submit_cmd(struct nvme_queue *nvmeq, ...@@ -297,6 +421,8 @@ static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
if (++tail == nvmeq->q_depth) if (++tail == nvmeq->q_depth)
tail = 0; tail = 0;
if (nvme_dbbuf_update_and_check_event(tail, nvmeq->dbbuf_sq_db,
nvmeq->dbbuf_sq_ei))
writel(tail, nvmeq->q_db); writel(tail, nvmeq->q_db);
nvmeq->sq_tail = tail; nvmeq->sq_tail = tail;
} }
...@@ -686,6 +812,8 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag) ...@@ -686,6 +812,8 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
return; return;
if (likely(nvmeq->cq_vector >= 0)) if (likely(nvmeq->cq_vector >= 0))
if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
nvmeq->dbbuf_cq_ei))
writel(head, nvmeq->q_db + nvmeq->dev->db_stride); writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
nvmeq->cq_head = head; nvmeq->cq_head = head;
nvmeq->cq_phase = phase; nvmeq->cq_phase = phase;
...@@ -718,10 +846,8 @@ static irqreturn_t nvme_irq_check(int irq, void *data) ...@@ -718,10 +846,8 @@ static irqreturn_t nvme_irq_check(int irq, void *data)
return IRQ_NONE; return IRQ_NONE;
} }
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag) static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)
{ {
struct nvme_queue *nvmeq = hctx->driver_data;
if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) { if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) {
spin_lock_irq(&nvmeq->q_lock); spin_lock_irq(&nvmeq->q_lock);
__nvme_process_cq(nvmeq, &tag); __nvme_process_cq(nvmeq, &tag);
...@@ -734,6 +860,13 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag) ...@@ -734,6 +860,13 @@ static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
return 0; return 0;
} }
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
{
struct nvme_queue *nvmeq = hctx->driver_data;
return __nvme_poll(nvmeq, tag);
}
static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl, int aer_idx) static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl, int aer_idx)
{ {
struct nvme_dev *dev = to_nvme_dev(ctrl); struct nvme_dev *dev = to_nvme_dev(ctrl);
...@@ -785,7 +918,7 @@ static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, ...@@ -785,7 +918,7 @@ static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
struct nvme_queue *nvmeq) struct nvme_queue *nvmeq)
{ {
struct nvme_command c; struct nvme_command c;
int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; int flags = NVME_QUEUE_PHYS_CONTIG;
/* /*
* Note: we (ab)use the fact the the prp fields survive if no data * Note: we (ab)use the fact the the prp fields survive if no data
...@@ -831,6 +964,16 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) ...@@ -831,6 +964,16 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
struct request *abort_req; struct request *abort_req;
struct nvme_command cmd; struct nvme_command cmd;
/*
* Did we miss an interrupt?
*/
if (__nvme_poll(nvmeq, req->tag)) {
dev_warn(dev->ctrl.device,
"I/O %d QID %d timeout, completion polled\n",
req->tag, nvmeq->qid);
return BLK_EH_HANDLED;
}
/* /*
* Shutdown immediately if controller times out while starting. The * Shutdown immediately if controller times out while starting. The
* reset work will see the pci device disabled when it gets the forced * reset work will see the pci device disabled when it gets the forced
...@@ -1070,6 +1213,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) ...@@ -1070,6 +1213,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
nvmeq->cq_phase = 1; nvmeq->cq_phase = 1;
nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth)); memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
nvme_dbbuf_init(dev, nvmeq, qid);
dev->online_queues++; dev->online_queues++;
spin_unlock_irq(&nvmeq->q_lock); spin_unlock_irq(&nvmeq->q_lock);
} }
...@@ -1542,6 +1686,8 @@ static int nvme_dev_add(struct nvme_dev *dev) ...@@ -1542,6 +1686,8 @@ static int nvme_dev_add(struct nvme_dev *dev)
if (blk_mq_alloc_tag_set(&dev->tagset)) if (blk_mq_alloc_tag_set(&dev->tagset))
return 0; return 0;
dev->ctrl.tagset = &dev->tagset; dev->ctrl.tagset = &dev->tagset;
nvme_dbbuf_set(dev);
} else { } else {
blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1); blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);
...@@ -1728,6 +1874,7 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl) ...@@ -1728,6 +1874,7 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
{ {
struct nvme_dev *dev = to_nvme_dev(ctrl); struct nvme_dev *dev = to_nvme_dev(ctrl);
nvme_dbbuf_dma_free(dev);
put_device(dev->dev); put_device(dev->dev);
if (dev->tagset.tags) if (dev->tagset.tags)
blk_mq_free_tag_set(&dev->tagset); blk_mq_free_tag_set(&dev->tagset);
...@@ -1795,6 +1942,13 @@ static void nvme_reset_work(struct work_struct *work) ...@@ -1795,6 +1942,13 @@ static void nvme_reset_work(struct work_struct *work)
dev->ctrl.opal_dev = NULL; dev->ctrl.opal_dev = NULL;
} }
if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) {
result = nvme_dbbuf_dma_alloc(dev);
if (result)
dev_warn(dev->dev,
"unable to allocate dma for dbbuf\n");
}
result = nvme_setup_io_queues(dev); result = nvme_setup_io_queues(dev);
if (result) if (result)
goto out; goto out;
......
...@@ -122,7 +122,15 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req) ...@@ -122,7 +122,15 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req)
struct nvmet_ctrl *ctrl = NULL; struct nvmet_ctrl *ctrl = NULL;
u16 status = 0; u16 status = 0;
d = kmap(sg_page(req->sg)) + req->sg->offset; d = kmalloc(sizeof(*d), GFP_KERNEL);
if (!d) {
status = NVME_SC_INTERNAL;
goto complete;
}
status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
if (status)
goto out;
/* zero out initial completion result, assign values as needed */ /* zero out initial completion result, assign values as needed */
req->rsp->result.u32 = 0; req->rsp->result.u32 = 0;
...@@ -158,7 +166,8 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req) ...@@ -158,7 +166,8 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req)
req->rsp->result.u16 = cpu_to_le16(ctrl->cntlid); req->rsp->result.u16 = cpu_to_le16(ctrl->cntlid);
out: out:
kunmap(sg_page(req->sg)); kfree(d);
complete:
nvmet_req_complete(req, status); nvmet_req_complete(req, status);
} }
...@@ -170,7 +179,15 @@ static void nvmet_execute_io_connect(struct nvmet_req *req) ...@@ -170,7 +179,15 @@ static void nvmet_execute_io_connect(struct nvmet_req *req)
u16 qid = le16_to_cpu(c->qid); u16 qid = le16_to_cpu(c->qid);
u16 status = 0; u16 status = 0;
d = kmap(sg_page(req->sg)) + req->sg->offset; d = kmalloc(sizeof(*d), GFP_KERNEL);
if (!d) {
status = NVME_SC_INTERNAL;
goto complete;
}
status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
if (status)
goto out;
/* zero out initial completion result, assign values as needed */ /* zero out initial completion result, assign values as needed */
req->rsp->result.u32 = 0; req->rsp->result.u32 = 0;
...@@ -205,7 +222,8 @@ static void nvmet_execute_io_connect(struct nvmet_req *req) ...@@ -205,7 +222,8 @@ static void nvmet_execute_io_connect(struct nvmet_req *req)
pr_info("adding queue %d to ctrl %d.\n", qid, ctrl->cntlid); pr_info("adding queue %d to ctrl %d.\n", qid, ctrl->cntlid);
out: out:
kunmap(sg_page(req->sg)); kfree(d);
complete:
nvmet_req_complete(req, status); nvmet_req_complete(req, status);
return; return;
......
This diff is collapsed.
...@@ -246,11 +246,19 @@ struct fcloop_lsreq { ...@@ -246,11 +246,19 @@ struct fcloop_lsreq {
struct fcloop_fcpreq { struct fcloop_fcpreq {
struct fcloop_tport *tport; struct fcloop_tport *tport;
struct nvmefc_fcp_req *fcpreq; struct nvmefc_fcp_req *fcpreq;
spinlock_t reqlock;
u16 status; u16 status;
bool active;
bool aborted;
struct work_struct work; struct work_struct work;
struct nvmefc_tgt_fcp_req tgt_fcp_req; struct nvmefc_tgt_fcp_req tgt_fcp_req;
}; };
struct fcloop_ini_fcpreq {
struct nvmefc_fcp_req *fcpreq;
struct fcloop_fcpreq *tfcp_req;
struct work_struct iniwork;
};
static inline struct fcloop_lsreq * static inline struct fcloop_lsreq *
tgt_ls_req_to_lsreq(struct nvmefc_tgt_ls_req *tgt_lsreq) tgt_ls_req_to_lsreq(struct nvmefc_tgt_ls_req *tgt_lsreq)
...@@ -341,7 +349,21 @@ fcloop_xmt_ls_rsp(struct nvmet_fc_target_port *tport, ...@@ -341,7 +349,21 @@ fcloop_xmt_ls_rsp(struct nvmet_fc_target_port *tport,
} }
/* /*
* FCP IO operation done. call back up initiator "done" flows. * FCP IO operation done by initiator abort.
* call back up initiator "done" flows.
*/
static void
fcloop_tgt_fcprqst_ini_done_work(struct work_struct *work)
{
struct fcloop_ini_fcpreq *inireq =
container_of(work, struct fcloop_ini_fcpreq, iniwork);
inireq->fcpreq->done(inireq->fcpreq);
}
/*
* FCP IO operation done by target completion.
* call back up initiator "done" flows.
*/ */
static void static void
fcloop_tgt_fcprqst_done_work(struct work_struct *work) fcloop_tgt_fcprqst_done_work(struct work_struct *work)
...@@ -349,12 +371,18 @@ fcloop_tgt_fcprqst_done_work(struct work_struct *work) ...@@ -349,12 +371,18 @@ fcloop_tgt_fcprqst_done_work(struct work_struct *work)
struct fcloop_fcpreq *tfcp_req = struct fcloop_fcpreq *tfcp_req =
container_of(work, struct fcloop_fcpreq, work); container_of(work, struct fcloop_fcpreq, work);
struct fcloop_tport *tport = tfcp_req->tport; struct fcloop_tport *tport = tfcp_req->tport;
struct nvmefc_fcp_req *fcpreq = tfcp_req->fcpreq; struct nvmefc_fcp_req *fcpreq;
if (tport->remoteport) { spin_lock(&tfcp_req->reqlock);
fcpreq = tfcp_req->fcpreq;
spin_unlock(&tfcp_req->reqlock);
if (tport->remoteport && fcpreq) {
fcpreq->status = tfcp_req->status; fcpreq->status = tfcp_req->status;
fcpreq->done(fcpreq); fcpreq->done(fcpreq);
} }
kfree(tfcp_req);
} }
...@@ -364,20 +392,25 @@ fcloop_fcp_req(struct nvme_fc_local_port *localport, ...@@ -364,20 +392,25 @@ fcloop_fcp_req(struct nvme_fc_local_port *localport,
void *hw_queue_handle, void *hw_queue_handle,
struct nvmefc_fcp_req *fcpreq) struct nvmefc_fcp_req *fcpreq)
{ {
struct fcloop_fcpreq *tfcp_req = fcpreq->private;
struct fcloop_rport *rport = remoteport->private; struct fcloop_rport *rport = remoteport->private;
struct fcloop_ini_fcpreq *inireq = fcpreq->private;
struct fcloop_fcpreq *tfcp_req;
int ret = 0; int ret = 0;
INIT_WORK(&tfcp_req->work, fcloop_tgt_fcprqst_done_work); if (!rport->targetport)
return -ECONNREFUSED;
if (!rport->targetport) { tfcp_req = kzalloc(sizeof(*tfcp_req), GFP_KERNEL);
tfcp_req->status = NVME_SC_FC_TRANSPORT_ERROR; if (!tfcp_req)
schedule_work(&tfcp_req->work); return -ENOMEM;
return ret;
}
inireq->fcpreq = fcpreq;
inireq->tfcp_req = tfcp_req;
INIT_WORK(&inireq->iniwork, fcloop_tgt_fcprqst_ini_done_work);
tfcp_req->fcpreq = fcpreq; tfcp_req->fcpreq = fcpreq;
tfcp_req->tport = rport->targetport->private; tfcp_req->tport = rport->targetport->private;
spin_lock_init(&tfcp_req->reqlock);
INIT_WORK(&tfcp_req->work, fcloop_tgt_fcprqst_done_work);
ret = nvmet_fc_rcv_fcp_req(rport->targetport, &tfcp_req->tgt_fcp_req, ret = nvmet_fc_rcv_fcp_req(rport->targetport, &tfcp_req->tgt_fcp_req,
fcpreq->cmdaddr, fcpreq->cmdlen); fcpreq->cmdaddr, fcpreq->cmdlen);
...@@ -444,31 +477,66 @@ fcloop_fcp_op(struct nvmet_fc_target_port *tgtport, ...@@ -444,31 +477,66 @@ fcloop_fcp_op(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *tgt_fcpreq) struct nvmefc_tgt_fcp_req *tgt_fcpreq)
{ {
struct fcloop_fcpreq *tfcp_req = tgt_fcp_req_to_fcpreq(tgt_fcpreq); struct fcloop_fcpreq *tfcp_req = tgt_fcp_req_to_fcpreq(tgt_fcpreq);
struct nvmefc_fcp_req *fcpreq = tfcp_req->fcpreq; struct nvmefc_fcp_req *fcpreq;
u32 rsplen = 0, xfrlen = 0; u32 rsplen = 0, xfrlen = 0;
int fcp_err = 0; int fcp_err = 0, active, aborted;
u8 op = tgt_fcpreq->op; u8 op = tgt_fcpreq->op;
spin_lock(&tfcp_req->reqlock);
fcpreq = tfcp_req->fcpreq;
active = tfcp_req->active;
aborted = tfcp_req->aborted;
tfcp_req->active = true;
spin_unlock(&tfcp_req->reqlock);
if (unlikely(active))
/* illegal - call while i/o active */
return -EALREADY;
if (unlikely(aborted)) {
/* target transport has aborted i/o prior */
spin_lock(&tfcp_req->reqlock);
tfcp_req->active = false;
spin_unlock(&tfcp_req->reqlock);
tgt_fcpreq->transferred_length = 0;
tgt_fcpreq->fcp_error = -ECANCELED;
tgt_fcpreq->done(tgt_fcpreq);
return 0;
}
/*
* if fcpreq is NULL, the I/O has been aborted (from
* initiator side). For the target side, act as if all is well
* but don't actually move data.
*/
switch (op) { switch (op) {
case NVMET_FCOP_WRITEDATA: case NVMET_FCOP_WRITEDATA:
xfrlen = tgt_fcpreq->transfer_length; xfrlen = tgt_fcpreq->transfer_length;
fcloop_fcp_copy_data(op, tgt_fcpreq->sg, fcpreq->first_sgl, if (fcpreq) {
tgt_fcpreq->offset, xfrlen); fcloop_fcp_copy_data(op, tgt_fcpreq->sg,
fcpreq->first_sgl, tgt_fcpreq->offset,
xfrlen);
fcpreq->transferred_length += xfrlen; fcpreq->transferred_length += xfrlen;
}
break; break;
case NVMET_FCOP_READDATA: case NVMET_FCOP_READDATA:
case NVMET_FCOP_READDATA_RSP: case NVMET_FCOP_READDATA_RSP:
xfrlen = tgt_fcpreq->transfer_length; xfrlen = tgt_fcpreq->transfer_length;
fcloop_fcp_copy_data(op, tgt_fcpreq->sg, fcpreq->first_sgl, if (fcpreq) {
tgt_fcpreq->offset, xfrlen); fcloop_fcp_copy_data(op, tgt_fcpreq->sg,
fcpreq->first_sgl, tgt_fcpreq->offset,
xfrlen);
fcpreq->transferred_length += xfrlen; fcpreq->transferred_length += xfrlen;
}
if (op == NVMET_FCOP_READDATA) if (op == NVMET_FCOP_READDATA)
break; break;
/* Fall-Thru to RSP handling */ /* Fall-Thru to RSP handling */
case NVMET_FCOP_RSP: case NVMET_FCOP_RSP:
if (fcpreq) {
rsplen = ((fcpreq->rsplen < tgt_fcpreq->rsplen) ? rsplen = ((fcpreq->rsplen < tgt_fcpreq->rsplen) ?
fcpreq->rsplen : tgt_fcpreq->rsplen); fcpreq->rsplen : tgt_fcpreq->rsplen);
memcpy(fcpreq->rspaddr, tgt_fcpreq->rspaddr, rsplen); memcpy(fcpreq->rspaddr, tgt_fcpreq->rspaddr, rsplen);
...@@ -476,30 +544,61 @@ fcloop_fcp_op(struct nvmet_fc_target_port *tgtport, ...@@ -476,30 +544,61 @@ fcloop_fcp_op(struct nvmet_fc_target_port *tgtport,
fcp_err = -E2BIG; fcp_err = -E2BIG;
fcpreq->rcv_rsplen = rsplen; fcpreq->rcv_rsplen = rsplen;
fcpreq->status = 0; fcpreq->status = 0;
}
tfcp_req->status = 0; tfcp_req->status = 0;
break; break;
case NVMET_FCOP_ABORT:
tfcp_req->status = NVME_SC_FC_TRANSPORT_ABORTED;
break;
default: default:
fcp_err = -EINVAL; fcp_err = -EINVAL;
break; break;
} }
spin_lock(&tfcp_req->reqlock);
tfcp_req->active = false;
spin_unlock(&tfcp_req->reqlock);
tgt_fcpreq->transferred_length = xfrlen; tgt_fcpreq->transferred_length = xfrlen;
tgt_fcpreq->fcp_error = fcp_err; tgt_fcpreq->fcp_error = fcp_err;
tgt_fcpreq->done(tgt_fcpreq); tgt_fcpreq->done(tgt_fcpreq);
if ((!fcp_err) && (op == NVMET_FCOP_RSP ||
op == NVMET_FCOP_READDATA_RSP ||
op == NVMET_FCOP_ABORT))
schedule_work(&tfcp_req->work);
return 0; return 0;
} }
static void
fcloop_tgt_fcp_abort(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *tgt_fcpreq)
{
struct fcloop_fcpreq *tfcp_req = tgt_fcp_req_to_fcpreq(tgt_fcpreq);
int active;
/*
* mark aborted only in case there were 2 threads in transport
* (one doing io, other doing abort) and only kills ops posted
* after the abort request
*/
spin_lock(&tfcp_req->reqlock);
active = tfcp_req->active;
tfcp_req->aborted = true;
spin_unlock(&tfcp_req->reqlock);
tfcp_req->status = NVME_SC_FC_TRANSPORT_ABORTED;
/*
* nothing more to do. If io wasn't active, the transport should
* immediately call the req_release. If it was active, the op
* will complete, and the lldd should call req_release.
*/
}
static void
fcloop_fcp_req_release(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *tgt_fcpreq)
{
struct fcloop_fcpreq *tfcp_req = tgt_fcp_req_to_fcpreq(tgt_fcpreq);
schedule_work(&tfcp_req->work);
}
static void static void
fcloop_ls_abort(struct nvme_fc_local_port *localport, fcloop_ls_abort(struct nvme_fc_local_port *localport,
struct nvme_fc_remote_port *remoteport, struct nvme_fc_remote_port *remoteport,
...@@ -513,6 +612,27 @@ fcloop_fcp_abort(struct nvme_fc_local_port *localport, ...@@ -513,6 +612,27 @@ fcloop_fcp_abort(struct nvme_fc_local_port *localport,
void *hw_queue_handle, void *hw_queue_handle,
struct nvmefc_fcp_req *fcpreq) struct nvmefc_fcp_req *fcpreq)
{ {
struct fcloop_rport *rport = remoteport->private;
struct fcloop_ini_fcpreq *inireq = fcpreq->private;
struct fcloop_fcpreq *tfcp_req = inireq->tfcp_req;
if (!tfcp_req)
/* abort has already been called */
return;
if (rport->targetport)
nvmet_fc_rcv_fcp_abort(rport->targetport,
&tfcp_req->tgt_fcp_req);
/* break initiator/target relationship for io */
spin_lock(&tfcp_req->reqlock);
inireq->tfcp_req = NULL;
tfcp_req->fcpreq = NULL;
spin_unlock(&tfcp_req->reqlock);
/* post the aborted io completion */
fcpreq->status = -ECANCELED;
schedule_work(&inireq->iniwork);
} }
static void static void
...@@ -563,20 +683,23 @@ struct nvme_fc_port_template fctemplate = { ...@@ -563,20 +683,23 @@ struct nvme_fc_port_template fctemplate = {
.local_priv_sz = sizeof(struct fcloop_lport), .local_priv_sz = sizeof(struct fcloop_lport),
.remote_priv_sz = sizeof(struct fcloop_rport), .remote_priv_sz = sizeof(struct fcloop_rport),
.lsrqst_priv_sz = sizeof(struct fcloop_lsreq), .lsrqst_priv_sz = sizeof(struct fcloop_lsreq),
.fcprqst_priv_sz = sizeof(struct fcloop_fcpreq), .fcprqst_priv_sz = sizeof(struct fcloop_ini_fcpreq),
}; };
struct nvmet_fc_target_template tgttemplate = { struct nvmet_fc_target_template tgttemplate = {
.targetport_delete = fcloop_targetport_delete, .targetport_delete = fcloop_targetport_delete,
.xmt_ls_rsp = fcloop_xmt_ls_rsp, .xmt_ls_rsp = fcloop_xmt_ls_rsp,
.fcp_op = fcloop_fcp_op, .fcp_op = fcloop_fcp_op,
.fcp_abort = fcloop_tgt_fcp_abort,
.fcp_req_release = fcloop_fcp_req_release,
.max_hw_queues = FCLOOP_HW_QUEUES, .max_hw_queues = FCLOOP_HW_QUEUES,
.max_sgl_segments = FCLOOP_SGL_SEGS, .max_sgl_segments = FCLOOP_SGL_SEGS,
.max_dif_sgl_segments = FCLOOP_SGL_SEGS, .max_dif_sgl_segments = FCLOOP_SGL_SEGS,
.dma_boundary = FCLOOP_DMABOUND_4G, .dma_boundary = FCLOOP_DMABOUND_4G,
/* optional features */ /* optional features */
.target_features = NVMET_FCTGTFEAT_READDATA_RSP | .target_features = NVMET_FCTGTFEAT_CMD_IN_ISR |
NVMET_FCTGTFEAT_NEEDS_CMD_CPUSCHED, NVMET_FCTGTFEAT_NEEDS_CMD_CPUSCHED |
NVMET_FCTGTFEAT_OPDONE_IN_ISR,
/* sizes of additional private data for data structures */ /* sizes of additional private data for data structures */
.target_priv_sz = sizeof(struct fcloop_tport), .target_priv_sz = sizeof(struct fcloop_tport),
}; };
......
...@@ -408,9 +408,7 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ...@@ -408,9 +408,7 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
if (phba->ktime_on) if (phba->ktime_on)
lpfc_nvmet_ktime(phba, ctxp); lpfc_nvmet_ktime(phba, ctxp);
#endif #endif
/* Let Abort cmpl repost the context */ /* lpfc_nvmet_xmt_fcp_release() will recycle the context */
if (!(ctxp->flag & LPFC_NVMET_ABORT_OP))
lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
} else { } else {
ctxp->entry_cnt++; ctxp->entry_cnt++;
start_clean = offsetof(struct lpfc_iocbq, wqe); start_clean = offsetof(struct lpfc_iocbq, wqe);
...@@ -544,27 +542,6 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, ...@@ -544,27 +542,6 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
} }
#endif #endif
if (rsp->op == NVMET_FCOP_ABORT) {
lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
"6103 Abort op: oxri x%x %d cnt %d\n",
ctxp->oxid, ctxp->state, ctxp->entry_cnt);
lpfc_nvmeio_data(phba, "NVMET FCP ABRT: "
"xri x%x state x%x cnt x%x\n",
ctxp->oxid, ctxp->state, ctxp->entry_cnt);
atomic_inc(&lpfc_nvmep->xmt_fcp_abort);
ctxp->entry_cnt++;
ctxp->flag |= LPFC_NVMET_ABORT_OP;
if (ctxp->flag & LPFC_NVMET_IO_INP)
lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
ctxp->oxid);
else
lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
ctxp->oxid);
return 0;
}
/* Sanity check */ /* Sanity check */
if (ctxp->state == LPFC_NVMET_STE_ABORT) { if (ctxp->state == LPFC_NVMET_STE_ABORT) {
atomic_inc(&lpfc_nvmep->xmt_fcp_drop); atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
...@@ -634,10 +611,75 @@ lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport) ...@@ -634,10 +611,75 @@ lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)
complete(&tport->tport_unreg_done); complete(&tport->tport_unreg_done);
} }
static void
lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *req)
{
struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
struct lpfc_nvmet_rcv_ctx *ctxp =
container_of(req, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
struct lpfc_hba *phba = ctxp->phba;
lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
"6103 Abort op: oxri x%x %d cnt %d\n",
ctxp->oxid, ctxp->state, ctxp->entry_cnt);
lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x state x%x cnt x%x\n",
ctxp->oxid, ctxp->state, ctxp->entry_cnt);
atomic_inc(&lpfc_nvmep->xmt_fcp_abort);
ctxp->entry_cnt++;
ctxp->flag |= LPFC_NVMET_ABORT_OP;
if (ctxp->flag & LPFC_NVMET_IO_INP)
lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
ctxp->oxid);
else
lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
ctxp->oxid);
}
static void
lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *rsp)
{
struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
struct lpfc_nvmet_rcv_ctx *ctxp =
container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
struct lpfc_hba *phba = ctxp->phba;
unsigned long flags;
bool aborting = false;
spin_lock_irqsave(&ctxp->ctxlock, flags);
if (ctxp->flag & LPFC_NVMET_ABORT_OP) {
aborting = true;
ctxp->flag |= LPFC_NVMET_CTX_RLS;
}
spin_unlock_irqrestore(&ctxp->ctxlock, flags);
if (aborting)
/* let the abort path do the real release */
return;
/* Sanity check */
if (ctxp->state != LPFC_NVMET_STE_DONE) {
atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
"6117 Bad state IO x%x aborted\n",
ctxp->oxid);
}
lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d\n", ctxp->oxid,
ctxp->state, 0);
lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
}
static struct nvmet_fc_target_template lpfc_tgttemplate = { static struct nvmet_fc_target_template lpfc_tgttemplate = {
.targetport_delete = lpfc_nvmet_targetport_delete, .targetport_delete = lpfc_nvmet_targetport_delete,
.xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp, .xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp,
.fcp_op = lpfc_nvmet_xmt_fcp_op, .fcp_op = lpfc_nvmet_xmt_fcp_op,
.fcp_abort = lpfc_nvmet_xmt_fcp_abort,
.fcp_req_release = lpfc_nvmet_xmt_fcp_release,
.max_hw_queues = 1, .max_hw_queues = 1,
.max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS, .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
...@@ -669,7 +711,9 @@ lpfc_nvmet_create_targetport(struct lpfc_hba *phba) ...@@ -669,7 +711,9 @@ lpfc_nvmet_create_targetport(struct lpfc_hba *phba)
lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel; lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel;
lpfc_tgttemplate.max_sgl_segments = phba->cfg_sg_seg_cnt; lpfc_tgttemplate.max_sgl_segments = phba->cfg_sg_seg_cnt;
lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP | lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP |
NVMET_FCTGTFEAT_NEEDS_CMD_CPUSCHED; NVMET_FCTGTFEAT_NEEDS_CMD_CPUSCHED |
NVMET_FCTGTFEAT_CMD_IN_ISR |
NVMET_FCTGTFEAT_OPDONE_IN_ISR;
#if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate, error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate,
...@@ -832,6 +876,7 @@ lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, ...@@ -832,6 +876,7 @@ lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
ctxp->wqeq = NULL; ctxp->wqeq = NULL;
ctxp->state = LPFC_NVMET_STE_RCV; ctxp->state = LPFC_NVMET_STE_RCV;
ctxp->rqb_buffer = (void *)nvmebuf; ctxp->rqb_buffer = (void *)nvmebuf;
spin_lock_init(&ctxp->ctxlock);
lpfc_nvmeio_data(phba, "NVMET LS RCV: xri x%x sz %d from %06x\n", lpfc_nvmeio_data(phba, "NVMET LS RCV: xri x%x sz %d from %06x\n",
oxid, size, sid); oxid, size, sid);
...@@ -1593,6 +1638,8 @@ lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ...@@ -1593,6 +1638,8 @@ lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
struct lpfc_nvmet_rcv_ctx *ctxp; struct lpfc_nvmet_rcv_ctx *ctxp;
struct lpfc_nvmet_tgtport *tgtp; struct lpfc_nvmet_tgtport *tgtp;
uint32_t status, result; uint32_t status, result;
unsigned long flags;
bool released = false;
ctxp = cmdwqe->context2; ctxp = cmdwqe->context2;
status = bf_get(lpfc_wcqe_c_status, wcqe); status = bf_get(lpfc_wcqe_c_status, wcqe);
...@@ -1607,6 +1654,17 @@ lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ...@@ -1607,6 +1654,17 @@ lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
result, wcqe->word3); result, wcqe->word3);
ctxp->state = LPFC_NVMET_STE_DONE; ctxp->state = LPFC_NVMET_STE_DONE;
spin_lock_irqsave(&ctxp->ctxlock, flags);
if (ctxp->flag & LPFC_NVMET_CTX_RLS)
released = true;
ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
spin_unlock_irqrestore(&ctxp->ctxlock, flags);
/*
* if transport has released ctx, then can reuse it. Otherwise,
* will be recycled by transport release call.
*/
if (released)
lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf); lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
cmdwqe->context2 = NULL; cmdwqe->context2 = NULL;
...@@ -1630,7 +1688,9 @@ lpfc_nvmet_xmt_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ...@@ -1630,7 +1688,9 @@ lpfc_nvmet_xmt_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
{ {
struct lpfc_nvmet_rcv_ctx *ctxp; struct lpfc_nvmet_rcv_ctx *ctxp;
struct lpfc_nvmet_tgtport *tgtp; struct lpfc_nvmet_tgtport *tgtp;
unsigned long flags;
uint32_t status, result; uint32_t status, result;
bool released = false;
ctxp = cmdwqe->context2; ctxp = cmdwqe->context2;
status = bf_get(lpfc_wcqe_c_status, wcqe); status = bf_get(lpfc_wcqe_c_status, wcqe);
...@@ -1652,7 +1712,19 @@ lpfc_nvmet_xmt_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ...@@ -1652,7 +1712,19 @@ lpfc_nvmet_xmt_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
ctxp->state, ctxp->oxid); ctxp->state, ctxp->oxid);
} }
ctxp->state = LPFC_NVMET_STE_DONE; ctxp->state = LPFC_NVMET_STE_DONE;
spin_lock_irqsave(&ctxp->ctxlock, flags);
if (ctxp->flag & LPFC_NVMET_CTX_RLS)
released = true;
ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
spin_unlock_irqrestore(&ctxp->ctxlock, flags);
/*
* if transport has released ctx, then can reuse it. Otherwise,
* will be recycled by transport release call.
*/
if (released)
lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf); lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
cmdwqe->context2 = NULL; cmdwqe->context2 = NULL;
cmdwqe->context3 = NULL; cmdwqe->context3 = NULL;
} }
......
...@@ -81,6 +81,7 @@ struct lpfc_nvmet_rcv_ctx { ...@@ -81,6 +81,7 @@ struct lpfc_nvmet_rcv_ctx {
struct lpfc_iocbq *wqeq; struct lpfc_iocbq *wqeq;
struct lpfc_iocbq *abort_wqeq; struct lpfc_iocbq *abort_wqeq;
dma_addr_t txrdy_phys; dma_addr_t txrdy_phys;
spinlock_t ctxlock; /* protect flag access */
uint32_t *txrdy; uint32_t *txrdy;
uint32_t sid; uint32_t sid;
uint32_t offset; uint32_t offset;
...@@ -97,8 +98,10 @@ struct lpfc_nvmet_rcv_ctx { ...@@ -97,8 +98,10 @@ struct lpfc_nvmet_rcv_ctx {
#define LPFC_NVMET_STE_RSP 4 #define LPFC_NVMET_STE_RSP 4
#define LPFC_NVMET_STE_DONE 5 #define LPFC_NVMET_STE_DONE 5
uint16_t flag; uint16_t flag;
#define LPFC_NVMET_IO_INP 1 #define LPFC_NVMET_IO_INP 0x1
#define LPFC_NVMET_ABORT_OP 2 #define LPFC_NVMET_ABORT_OP 0x2
#define LPFC_NVMET_CTX_RLS 0x4
struct rqb_dmabuf *rqb_buffer; struct rqb_dmabuf *rqb_buffer;
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
......
...@@ -533,9 +533,6 @@ enum { ...@@ -533,9 +533,6 @@ enum {
* rsp as well * rsp as well
*/ */
NVMET_FCOP_RSP = 4, /* send rsp frame */ NVMET_FCOP_RSP = 4, /* send rsp frame */
NVMET_FCOP_ABORT = 5, /* abort exchange via ABTS */
NVMET_FCOP_BA_ACC = 6, /* send BA_ACC */
NVMET_FCOP_BA_RJT = 7, /* send BA_RJT */
}; };
/** /**
...@@ -572,8 +569,6 @@ enum { ...@@ -572,8 +569,6 @@ enum {
* upon compeletion of the operation. The nvmet-fc layer will also set a * upon compeletion of the operation. The nvmet-fc layer will also set a
* private pointer for its own use in the done routine. * private pointer for its own use in the done routine.
* *
* Note: the LLDD must never fail a NVMET_FCOP_ABORT request !!
*
* Values set by the NVMET-FC layer prior to calling the LLDD fcp_op * Values set by the NVMET-FC layer prior to calling the LLDD fcp_op
* entrypoint. * entrypoint.
* @op: Indicates the FCP IU operation to perform (see NVMET_FCOP_xxx) * @op: Indicates the FCP IU operation to perform (see NVMET_FCOP_xxx)
...@@ -655,6 +650,22 @@ enum { ...@@ -655,6 +650,22 @@ enum {
* on. The transport should pick a cpu to schedule the work * on. The transport should pick a cpu to schedule the work
* on. * on.
*/ */
NVMET_FCTGTFEAT_CMD_IN_ISR = (1 << 2),
/* Bit 2: When 0, the LLDD is calling the cmd rcv handler
* in a non-isr context, allowing the transport to finish
* op completion in the calling context. When 1, the LLDD
* is calling the cmd rcv handler in an ISR context,
* requiring the transport to transition to a workqueue
* for op completion.
*/
NVMET_FCTGTFEAT_OPDONE_IN_ISR = (1 << 3),
/* Bit 3: When 0, the LLDD is calling the op done handler
* in a non-isr context, allowing the transport to finish
* op completion in the calling context. When 1, the LLDD
* is calling the op done handler in an ISR context,
* requiring the transport to transition to a workqueue
* for op completion.
*/
}; };
...@@ -725,12 +736,12 @@ struct nvmet_fc_target_port { ...@@ -725,12 +736,12 @@ struct nvmet_fc_target_port {
* be freed/released. * be freed/released.
* Entrypoint is Mandatory. * Entrypoint is Mandatory.
* *
* @fcp_op: Called to perform a data transfer, transmit a response, or * @fcp_op: Called to perform a data transfer or transmit a response.
* abort an FCP opertion. The nvmefc_tgt_fcp_req structure is the same * The nvmefc_tgt_fcp_req structure is the same LLDD-supplied
* LLDD-supplied exchange structure specified in the * exchange structure specified in the nvmet_fc_rcv_fcp_req() call
* nvmet_fc_rcv_fcp_req() call made when the FCP CMD IU was received. * made when the FCP CMD IU was received. The op field in the
* The op field in the structure shall indicate the operation for * structure shall indicate the operation for the LLDD to perform
* the LLDD to perform relative to the io. * relative to the io.
* NVMET_FCOP_READDATA operation: the LLDD is to send the * NVMET_FCOP_READDATA operation: the LLDD is to send the
* payload data (described by sglist) to the host in 1 or * payload data (described by sglist) to the host in 1 or
* more FC sequences (preferrably 1). Note: the fc-nvme layer * more FC sequences (preferrably 1). Note: the fc-nvme layer
...@@ -752,29 +763,31 @@ struct nvmet_fc_target_port { ...@@ -752,29 +763,31 @@ struct nvmet_fc_target_port {
* successfully, the LLDD is to update the nvmefc_tgt_fcp_req * successfully, the LLDD is to update the nvmefc_tgt_fcp_req
* transferred_length field and may subsequently transmit the * transferred_length field and may subsequently transmit the
* FCP_RSP iu payload (described by rspbuf, rspdma, rsplen). * FCP_RSP iu payload (described by rspbuf, rspdma, rsplen).
* The LLDD is to await FCP_CONF reception to confirm the RSP * If FCP_CONF is supported, the LLDD is to await FCP_CONF
* reception by the host. The LLDD may retramsit the FCP_RSP iu * reception to confirm the RSP reception by the host. The LLDD
* if necessary per FC-NVME. Upon reception of FCP_CONF, or upon * may retramsit the FCP_RSP iu if necessary per FC-NVME. Upon
* FCP_CONF failure, the LLDD is to set the nvmefc_tgt_fcp_req * transmission of the FCP_RSP iu if FCP_CONF is not supported,
* fcp_error field and consider the operation complete.. * or upon success/failure of FCP_CONF if it is supported, the
* LLDD is to set the nvmefc_tgt_fcp_req fcp_error field and
* consider the operation complete.
* NVMET_FCOP_RSP: the LLDD is to transmit the FCP_RSP iu payload * NVMET_FCOP_RSP: the LLDD is to transmit the FCP_RSP iu payload
* (described by rspbuf, rspdma, rsplen). The LLDD is to await * (described by rspbuf, rspdma, rsplen). If FCP_CONF is
* FCP_CONF reception to confirm the RSP reception by the host. * supported, the LLDD is to await FCP_CONF reception to confirm
* The LLDD may retramsit the FCP_RSP iu if necessary per FC-NVME. * the RSP reception by the host. The LLDD may retramsit the
* Upon reception of FCP_CONF, or upon FCP_CONF failure, the * FCP_RSP iu if FCP_CONF is not received per FC-NVME. Upon
* transmission of the FCP_RSP iu if FCP_CONF is not supported,
* or upon success/failure of FCP_CONF if it is supported, the
* LLDD is to set the nvmefc_tgt_fcp_req fcp_error field and * LLDD is to set the nvmefc_tgt_fcp_req fcp_error field and
* consider the operation complete.. * consider the operation complete.
* NVMET_FCOP_ABORT: the LLDD is to terminate the exchange
* corresponding to the fcp operation. The LLDD shall send
* ABTS and follow FC exchange abort-multi rules, including
* ABTS retries and possible logout.
* Upon completing the indicated operation, the LLDD is to set the * Upon completing the indicated operation, the LLDD is to set the
* status fields for the operation (tranferred_length and fcp_error * status fields for the operation (tranferred_length and fcp_error
* status) in the request, then all the "done" routine * status) in the request, then call the "done" routine
* indicated in the fcp request. Upon return from the "done" * indicated in the fcp request. After the operation completes,
* routine for either a NVMET_FCOP_RSP or NVMET_FCOP_ABORT operation * regardless of whether the FCP_RSP iu was successfully transmit,
* the fc-nvme layer will not longer reference the fcp request, * the LLDD-supplied exchange structure must remain valid until the
* allowing the LLDD to free/release the fcp request. * transport calls the fcp_req_release() callback to return ownership
* of the exchange structure back to the LLDD so that it may be used
* for another fcp command.
* Note: when calling the done routine for READDATA or WRITEDATA * Note: when calling the done routine for READDATA or WRITEDATA
* operations, the fc-nvme layer may immediate convert, in the same * operations, the fc-nvme layer may immediate convert, in the same
* thread and before returning to the LLDD, the fcp operation to * thread and before returning to the LLDD, the fcp operation to
...@@ -786,6 +799,22 @@ struct nvmet_fc_target_port { ...@@ -786,6 +799,22 @@ struct nvmet_fc_target_port {
* Returns 0 on success, -<errno> on failure (Ex: -EIO) * Returns 0 on success, -<errno> on failure (Ex: -EIO)
* Entrypoint is Mandatory. * Entrypoint is Mandatory.
* *
* @fcp_abort: Called by the transport to abort an active command.
* The command may be in-between operations (nothing active in LLDD)
* or may have an active WRITEDATA operation pending. The LLDD is to
* initiate the ABTS process for the command and return from the
* callback. The ABTS does not need to be complete on the command.
* The fcp_abort callback inherently cannot fail. After the
* fcp_abort() callback completes, the transport will wait for any
* outstanding operation (if there was one) to complete, then will
* call the fcp_req_release() callback to return the command's
* exchange context back to the LLDD.
*
* @fcp_req_release: Called by the transport to return a nvmefc_tgt_fcp_req
* to the LLDD after all operations on the fcp operation are complete.
* This may be due to the command completing or upon completion of
* abort cleanup.
*
* @max_hw_queues: indicates the maximum number of hw queues the LLDD * @max_hw_queues: indicates the maximum number of hw queues the LLDD
* supports for cpu affinitization. * supports for cpu affinitization.
* Value is Mandatory. Must be at least 1. * Value is Mandatory. Must be at least 1.
...@@ -820,7 +849,11 @@ struct nvmet_fc_target_template { ...@@ -820,7 +849,11 @@ struct nvmet_fc_target_template {
int (*xmt_ls_rsp)(struct nvmet_fc_target_port *tgtport, int (*xmt_ls_rsp)(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_ls_req *tls_req); struct nvmefc_tgt_ls_req *tls_req);
int (*fcp_op)(struct nvmet_fc_target_port *tgtport, int (*fcp_op)(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *); struct nvmefc_tgt_fcp_req *fcpreq);
void (*fcp_abort)(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *fcpreq);
void (*fcp_req_release)(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *fcpreq);
u32 max_hw_queues; u32 max_hw_queues;
u16 max_sgl_segments; u16 max_sgl_segments;
...@@ -848,4 +881,7 @@ int nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *tgtport, ...@@ -848,4 +881,7 @@ int nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *fcpreq, struct nvmefc_tgt_fcp_req *fcpreq,
void *cmdiubuf, u32 cmdiubuf_len); void *cmdiubuf, u32 cmdiubuf_len);
void nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *fcpreq);
#endif /* _NVME_FC_DRIVER_H */ #endif /* _NVME_FC_DRIVER_H */
...@@ -245,6 +245,7 @@ enum { ...@@ -245,6 +245,7 @@ enum {
NVME_CTRL_ONCS_WRITE_ZEROES = 1 << 3, NVME_CTRL_ONCS_WRITE_ZEROES = 1 << 3,
NVME_CTRL_VWC_PRESENT = 1 << 0, NVME_CTRL_VWC_PRESENT = 1 << 0,
NVME_CTRL_OACS_SEC_SUPP = 1 << 0, NVME_CTRL_OACS_SEC_SUPP = 1 << 0,
NVME_CTRL_OACS_DBBUF_SUPP = 1 << 7,
}; };
struct nvme_lbaf { struct nvme_lbaf {
...@@ -603,6 +604,7 @@ enum nvme_admin_opcode { ...@@ -603,6 +604,7 @@ enum nvme_admin_opcode {
nvme_admin_download_fw = 0x11, nvme_admin_download_fw = 0x11,
nvme_admin_ns_attach = 0x15, nvme_admin_ns_attach = 0x15,
nvme_admin_keep_alive = 0x18, nvme_admin_keep_alive = 0x18,
nvme_admin_dbbuf = 0x7C,
nvme_admin_format_nvm = 0x80, nvme_admin_format_nvm = 0x80,
nvme_admin_security_send = 0x81, nvme_admin_security_send = 0x81,
nvme_admin_security_recv = 0x82, nvme_admin_security_recv = 0x82,
...@@ -874,6 +876,16 @@ struct nvmf_property_get_command { ...@@ -874,6 +876,16 @@ struct nvmf_property_get_command {
__u8 resv4[16]; __u8 resv4[16];
}; };
struct nvme_dbbuf {
__u8 opcode;
__u8 flags;
__u16 command_id;
__u32 rsvd1[5];
__le64 prp1;
__le64 prp2;
__u32 rsvd12[6];
};
struct nvme_command { struct nvme_command {
union { union {
struct nvme_common_command common; struct nvme_common_command common;
...@@ -893,6 +905,7 @@ struct nvme_command { ...@@ -893,6 +905,7 @@ struct nvme_command {
struct nvmf_connect_command connect; struct nvmf_connect_command connect;
struct nvmf_property_set_command prop_set; struct nvmf_property_set_command prop_set;
struct nvmf_property_get_command prop_get; struct nvmf_property_get_command prop_get;
struct nvme_dbbuf dbbuf;
}; };
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment