Commit cdbb0a4f authored by Santosh Vernekar's avatar Santosh Vernekar Committed by James Bottomley

[SCSI] qla2xxx: Handle outstanding mbx cmds on hung f/w scenarios.

Outstanding mailbox commands, have no way to recover on f/w hung, and we
timeout on waiting for mbx response. This in turn affects the recovery process
as follows:
- We might already be in dpc while waiting for mbx to complete, so recovery for
that pci function will never get invoked. Reset Timeout (10 sec) is far less
than mbx timeout (30 sec).
- Other mbx cmds will get stuck due to serial mbx access.

Solution is to identify fw-hung scenario and handle outstanding mbx commands to
have an early-exit instead of waiting for response.
Other mbx commands waiting for access will also do an early-exit if fw-hung is
still applicable.
Signed-off-by: default avatarGiridhar Malavali <giridhar.malavali@qlogic.com>
Signed-off-by: default avatarJames Bottomley <James.Bottomley@suse.de>
parent 3f3b6f98
...@@ -2413,6 +2413,7 @@ struct qla_hw_data { ...@@ -2413,6 +2413,7 @@ struct qla_hw_data {
uint32_t cpu_affinity_enabled :1; uint32_t cpu_affinity_enabled :1;
uint32_t disable_msix_handshake :1; uint32_t disable_msix_handshake :1;
uint32_t fcp_prio_enabled :1; uint32_t fcp_prio_enabled :1;
uint32_t fw_hung :1;
} flags; } flags;
/* This spinlock is used to protect "io transactions", you must /* This spinlock is used to protect "io transactions", you must
......
...@@ -1972,7 +1972,8 @@ qla2x00_fw_ready(scsi_qla_host_t *vha) ...@@ -1972,7 +1972,8 @@ qla2x00_fw_ready(scsi_qla_host_t *vha)
} }
} else { } else {
/* Mailbox cmd failed. Timeout on min_wait. */ /* Mailbox cmd failed. Timeout on min_wait. */
if (time_after_eq(jiffies, mtime)) if (time_after_eq(jiffies, mtime) ||
(IS_QLA82XX(ha) && ha->flags.fw_hung))
break; break;
} }
......
...@@ -37,7 +37,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp) ...@@ -37,7 +37,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
device_reg_t __iomem *reg; device_reg_t __iomem *reg;
uint8_t abort_active; uint8_t abort_active;
uint8_t io_lock_on; uint8_t io_lock_on;
uint16_t command; uint16_t command = 0;
uint16_t *iptr; uint16_t *iptr;
uint16_t __iomem *optr; uint16_t __iomem *optr;
uint32_t cnt; uint32_t cnt;
...@@ -83,6 +83,13 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp) ...@@ -83,6 +83,13 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
return QLA_FUNCTION_TIMEOUT; return QLA_FUNCTION_TIMEOUT;
} }
if (IS_QLA82XX(ha) && ha->flags.fw_hung) {
/* Setting Link-Down error */
mcp->mb[0] = MBS_LINK_DOWN_ERROR;
rval = QLA_FUNCTION_FAILED;
goto premature_exit;
}
ha->flags.mbox_busy = 1; ha->flags.mbox_busy = 1;
/* Save mailbox command for debug */ /* Save mailbox command for debug */
ha->mcp = mcp; ha->mcp = mcp;
...@@ -151,7 +158,8 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp) ...@@ -151,7 +158,8 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
DEBUG2_3_11(printk(KERN_INFO DEBUG2_3_11(printk(KERN_INFO
"%s(%ld): Pending Mailbox timeout. " "%s(%ld): Pending Mailbox timeout. "
"Exiting.\n", __func__, base_vha->host_no)); "Exiting.\n", __func__, base_vha->host_no));
return QLA_FUNCTION_TIMEOUT; rval = QLA_FUNCTION_TIMEOUT;
goto premature_exit;
} }
WRT_REG_DWORD(&reg->isp82.hint, HINT_MBX_INT_PENDING); WRT_REG_DWORD(&reg->isp82.hint, HINT_MBX_INT_PENDING);
} else if (IS_FWI2_CAPABLE(ha)) } else if (IS_FWI2_CAPABLE(ha))
...@@ -176,7 +184,8 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp) ...@@ -176,7 +184,8 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
DEBUG2_3_11(printk(KERN_INFO DEBUG2_3_11(printk(KERN_INFO
"%s(%ld): Pending Mailbox timeout. " "%s(%ld): Pending Mailbox timeout. "
"Exiting.\n", __func__, base_vha->host_no)); "Exiting.\n", __func__, base_vha->host_no));
return QLA_FUNCTION_TIMEOUT; rval = QLA_FUNCTION_TIMEOUT;
goto premature_exit;
} }
WRT_REG_DWORD(&reg->isp82.hint, HINT_MBX_INT_PENDING); WRT_REG_DWORD(&reg->isp82.hint, HINT_MBX_INT_PENDING);
} else if (IS_FWI2_CAPABLE(ha)) } else if (IS_FWI2_CAPABLE(ha))
...@@ -214,6 +223,15 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp) ...@@ -214,6 +223,15 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
ha->flags.mbox_int = 0; ha->flags.mbox_int = 0;
clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags); clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
if (IS_QLA82XX(ha) && ha->flags.fw_hung) {
ha->flags.mbox_busy = 0;
/* Setting Link-Down error */
mcp->mb[0] = MBS_LINK_DOWN_ERROR;
ha->mcp = NULL;
rval = QLA_FUNCTION_FAILED;
goto premature_exit;
}
if (ha->mailbox_out[0] != MBS_COMMAND_COMPLETE) if (ha->mailbox_out[0] != MBS_COMMAND_COMPLETE)
rval = QLA_FUNCTION_FAILED; rval = QLA_FUNCTION_FAILED;
...@@ -279,35 +297,51 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp) ...@@ -279,35 +297,51 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
DEBUG2_3_11(printk("%s(%ld): timeout schedule " DEBUG2_3_11(printk("%s(%ld): timeout schedule "
"isp_abort_needed.\n", __func__, "isp_abort_needed.\n", __func__,
base_vha->host_no)); base_vha->host_no));
qla_printk(KERN_WARNING, ha,
"Mailbox command timeout occurred. Scheduling ISP " if (!test_bit(ISP_ABORT_NEEDED, &vha->dpc_flags) &&
"abort. eeh_busy: 0x%x\n", ha->flags.eeh_busy); !test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags) &&
set_bit(ISP_ABORT_NEEDED, &base_vha->dpc_flags); !test_bit(ISP_ABORT_RETRY, &vha->dpc_flags)) {
qla2xxx_wake_dpc(vha);
qla_printk(KERN_WARNING, ha,
"Mailbox command timeout occured. "
"Scheduling ISP " "abort. eeh_busy: 0x%x\n",
ha->flags.eeh_busy);
set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
qla2xxx_wake_dpc(vha);
}
} else if (!abort_active) { } else if (!abort_active) {
/* call abort directly since we are in the DPC thread */ /* call abort directly since we are in the DPC thread */
DEBUG(printk("%s(%ld): timeout calling abort_isp\n", DEBUG(printk("%s(%ld): timeout calling abort_isp\n",
__func__, base_vha->host_no)); __func__, base_vha->host_no));
DEBUG2_3_11(printk("%s(%ld): timeout calling " DEBUG2_3_11(printk("%s(%ld): timeout calling "
"abort_isp\n", __func__, base_vha->host_no)); "abort_isp\n", __func__, base_vha->host_no));
qla_printk(KERN_WARNING, ha,
"Mailbox command timeout occurred. Issuing ISP " if (!test_bit(ISP_ABORT_NEEDED, &vha->dpc_flags) &&
"abort.\n"); !test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags) &&
!test_bit(ISP_ABORT_RETRY, &vha->dpc_flags)) {
set_bit(ABORT_ISP_ACTIVE, &base_vha->dpc_flags);
clear_bit(ISP_ABORT_NEEDED, &base_vha->dpc_flags); qla_printk(KERN_WARNING, ha,
if (ha->isp_ops->abort_isp(base_vha)) { "Mailbox command timeout occured. "
/* Failed. retry later. */ "Issuing ISP abort.\n");
set_bit(ISP_ABORT_NEEDED, &base_vha->dpc_flags);
set_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags);
clear_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
if (ha->isp_ops->abort_isp(vha)) {
/* Failed. retry later. */
set_bit(ISP_ABORT_NEEDED,
&vha->dpc_flags);
}
clear_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags);
DEBUG(printk("%s(%ld): finished abort_isp\n",
__func__, vha->host_no));
DEBUG2_3_11(printk(
"%s(%ld): finished abort_isp\n",
__func__, vha->host_no));
} }
clear_bit(ABORT_ISP_ACTIVE, &base_vha->dpc_flags);
DEBUG(printk("%s(%ld): finished abort_isp\n", __func__,
base_vha->host_no));
DEBUG2_3_11(printk("%s(%ld): finished abort_isp\n",
__func__, base_vha->host_no));
} }
} }
premature_exit:
/* Allow next mbx cmd to come in. */ /* Allow next mbx cmd to come in. */
complete(&ha->mbx_cmd_comp); complete(&ha->mbx_cmd_comp);
......
...@@ -3543,6 +3543,14 @@ qla82xx_check_fw_alive(scsi_qla_host_t *vha) ...@@ -3543,6 +3543,14 @@ qla82xx_check_fw_alive(scsi_qla_host_t *vha)
set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags); set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
} }
qla2xxx_wake_dpc(vha); qla2xxx_wake_dpc(vha);
if (ha->flags.mbox_busy) {
ha->flags.fw_hung = 1;
ha->flags.mbox_int = 1;
DEBUG2(qla_printk(KERN_ERR, ha,
"Due to fw hung, doing premature "
"completion of mbx command\n"));
complete(&ha->mbx_intr_comp);
}
} }
} }
vha->fw_heartbeat_counter = fw_heartbeat_counter; vha->fw_heartbeat_counter = fw_heartbeat_counter;
...@@ -3646,6 +3654,14 @@ void qla82xx_watchdog(scsi_qla_host_t *vha) ...@@ -3646,6 +3654,14 @@ void qla82xx_watchdog(scsi_qla_host_t *vha)
"%s(): Adapter reset needed!\n", __func__); "%s(): Adapter reset needed!\n", __func__);
set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags); set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
qla2xxx_wake_dpc(vha); qla2xxx_wake_dpc(vha);
if (ha->flags.mbox_busy) {
ha->flags.fw_hung = 1;
ha->flags.mbox_int = 1;
DEBUG2(qla_printk(KERN_ERR, ha,
"Need reset, doing premature "
"completion of mbx command\n"));
complete(&ha->mbx_intr_comp);
}
} else { } else {
qla82xx_check_fw_alive(vha); qla82xx_check_fw_alive(vha);
} }
...@@ -3701,8 +3717,10 @@ qla82xx_abort_isp(scsi_qla_host_t *vha) ...@@ -3701,8 +3717,10 @@ qla82xx_abort_isp(scsi_qla_host_t *vha)
qla82xx_clear_rst_ready(ha); qla82xx_clear_rst_ready(ha);
qla82xx_idc_unlock(ha); qla82xx_idc_unlock(ha);
if (rval == QLA_SUCCESS) if (rval == QLA_SUCCESS) {
ha->flags.fw_hung = 0;
qla82xx_restart_isp(vha); qla82xx_restart_isp(vha);
}
if (rval) { if (rval) {
vha->flags.online = 1; vha->flags.online = 1;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment