Commit 7cb5101e authored by Oded Gabbay's avatar Oded Gabbay

habanalabs: prevent host crash during suspend/resume

This patch fixes the implementation of suspend/resume of the device so that
upon resume of the device, the host won't crash due to PCI completion
timeout.

Upon suspend, the device is being reset due to PERST. Therefore, upon
resume, the driver must initialize the PCI controller as if the driver was
loaded. If the controller is not initialized and the device tries to
access the device through the PCI bars, the host will crash with PCI
completion timeout error.
Signed-off-by: default avatarOded Gabbay <oded.gabbay@gmail.com>
parent cbaa99ed
...@@ -416,6 +416,27 @@ int hl_device_suspend(struct hl_device *hdev) ...@@ -416,6 +416,27 @@ int hl_device_suspend(struct hl_device *hdev)
pci_save_state(hdev->pdev); pci_save_state(hdev->pdev);
/* Block future CS/VM/JOB completion operations */
rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
if (rc) {
dev_err(hdev->dev, "Can't suspend while in reset\n");
return -EIO;
}
/* This blocks all other stuff that is not blocked by in_reset */
hdev->disabled = true;
/*
* Flush anyone that is inside the critical section of enqueue
* jobs to the H/W
*/
hdev->asic_funcs->hw_queues_lock(hdev);
hdev->asic_funcs->hw_queues_unlock(hdev);
/* Flush processes that are sending message to CPU */
mutex_lock(&hdev->send_cpu_message_lock);
mutex_unlock(&hdev->send_cpu_message_lock);
rc = hdev->asic_funcs->suspend(hdev); rc = hdev->asic_funcs->suspend(hdev);
if (rc) if (rc)
dev_err(hdev->dev, dev_err(hdev->dev,
...@@ -443,21 +464,38 @@ int hl_device_resume(struct hl_device *hdev) ...@@ -443,21 +464,38 @@ int hl_device_resume(struct hl_device *hdev)
pci_set_power_state(hdev->pdev, PCI_D0); pci_set_power_state(hdev->pdev, PCI_D0);
pci_restore_state(hdev->pdev); pci_restore_state(hdev->pdev);
rc = pci_enable_device(hdev->pdev); rc = pci_enable_device_mem(hdev->pdev);
if (rc) { if (rc) {
dev_err(hdev->dev, dev_err(hdev->dev,
"Failed to enable PCI device in resume\n"); "Failed to enable PCI device in resume\n");
return rc; return rc;
} }
pci_set_master(hdev->pdev);
rc = hdev->asic_funcs->resume(hdev); rc = hdev->asic_funcs->resume(hdev);
if (rc) { if (rc) {
dev_err(hdev->dev, dev_err(hdev->dev, "Failed to resume device after suspend\n");
"Failed to enable PCI access from device CPU\n"); goto disable_device;
return rc; }
hdev->disabled = false;
atomic_set(&hdev->in_reset, 0);
rc = hl_device_reset(hdev, true, false);
if (rc) {
dev_err(hdev->dev, "Failed to reset device during resume\n");
goto disable_device;
} }
return 0; return 0;
disable_device:
pci_clear_master(hdev->pdev);
pci_disable_device(hdev->pdev);
return rc;
} }
static void hl_device_hard_reset_pending(struct work_struct *work) static void hl_device_hard_reset_pending(struct work_struct *work)
......
...@@ -1201,15 +1201,6 @@ static int goya_stop_external_queues(struct hl_device *hdev) ...@@ -1201,15 +1201,6 @@ static int goya_stop_external_queues(struct hl_device *hdev)
return retval; return retval;
} }
static void goya_resume_external_queues(struct hl_device *hdev)
{
WREG32(mmDMA_QM_0_GLBL_CFG1, 0);
WREG32(mmDMA_QM_1_GLBL_CFG1, 0);
WREG32(mmDMA_QM_2_GLBL_CFG1, 0);
WREG32(mmDMA_QM_3_GLBL_CFG1, 0);
WREG32(mmDMA_QM_4_GLBL_CFG1, 0);
}
/* /*
* goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
* *
...@@ -2178,36 +2169,6 @@ static int goya_stop_internal_queues(struct hl_device *hdev) ...@@ -2178,36 +2169,6 @@ static int goya_stop_internal_queues(struct hl_device *hdev)
return retval; return retval;
} }
static void goya_resume_internal_queues(struct hl_device *hdev)
{
WREG32(mmMME_QM_GLBL_CFG1, 0);
WREG32(mmMME_CMDQ_GLBL_CFG1, 0);
WREG32(mmTPC0_QM_GLBL_CFG1, 0);
WREG32(mmTPC0_CMDQ_GLBL_CFG1, 0);
WREG32(mmTPC1_QM_GLBL_CFG1, 0);
WREG32(mmTPC1_CMDQ_GLBL_CFG1, 0);
WREG32(mmTPC2_QM_GLBL_CFG1, 0);
WREG32(mmTPC2_CMDQ_GLBL_CFG1, 0);
WREG32(mmTPC3_QM_GLBL_CFG1, 0);
WREG32(mmTPC3_CMDQ_GLBL_CFG1, 0);
WREG32(mmTPC4_QM_GLBL_CFG1, 0);
WREG32(mmTPC4_CMDQ_GLBL_CFG1, 0);
WREG32(mmTPC5_QM_GLBL_CFG1, 0);
WREG32(mmTPC5_CMDQ_GLBL_CFG1, 0);
WREG32(mmTPC6_QM_GLBL_CFG1, 0);
WREG32(mmTPC6_CMDQ_GLBL_CFG1, 0);
WREG32(mmTPC7_QM_GLBL_CFG1, 0);
WREG32(mmTPC7_CMDQ_GLBL_CFG1, 0);
}
static void goya_dma_stall(struct hl_device *hdev) static void goya_dma_stall(struct hl_device *hdev)
{ {
WREG32(mmDMA_QM_0_GLBL_CFG1, 1 << DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT); WREG32(mmDMA_QM_0_GLBL_CFG1, 1 << DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT);
...@@ -2905,20 +2866,6 @@ int goya_suspend(struct hl_device *hdev) ...@@ -2905,20 +2866,6 @@ int goya_suspend(struct hl_device *hdev)
{ {
int rc; int rc;
rc = goya_stop_internal_queues(hdev);
if (rc) {
dev_err(hdev->dev, "failed to stop internal queues\n");
return rc;
}
rc = goya_stop_external_queues(hdev);
if (rc) {
dev_err(hdev->dev, "failed to stop external queues\n");
return rc;
}
rc = goya_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS); rc = goya_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
if (rc) if (rc)
dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
...@@ -2928,15 +2875,7 @@ int goya_suspend(struct hl_device *hdev) ...@@ -2928,15 +2875,7 @@ int goya_suspend(struct hl_device *hdev)
int goya_resume(struct hl_device *hdev) int goya_resume(struct hl_device *hdev)
{ {
int rc; return goya_init_iatu(hdev);
goya_resume_external_queues(hdev);
goya_resume_internal_queues(hdev);
rc = goya_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS);
if (rc)
dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
return rc;
} }
static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma, static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment