ASoC: SOF: Re-visit firmware state and panic tracking/handling

Merge series from Peter Ujfalusi <peter.ujfalusi@linux.intel.com>:

this series will improve how we are tracking the firmware's state to be able to
avoid communication with it when it is not going to answer due to a panic and
we will attempt to force power cycle the DSP to recover at the next runtime
suspend time.

The state handling brings in other improvements on the way the kernel reports
errors and DSP panics to reduce the printed lines for normal users, but at the
same time allowing developers (or for bug reports) to have more precise
information available to track down the issue.

We can now place messages easily in the correct debug level and not bound to the
static ERROR for some of the print chains, causing excess amount or partial
information to be printed, confusing users and machines (CI).

I would have prefered to split this series up, but it was developed together to
achieve a single goal to reduce the noise, but also provide the details we need
to be able to rootcause issues.

include/sound/sof.h

@@ -17,6 +17,28 @@
 
 struct snd_sof_dsp_ops;
 
+/**
+ * enum sof_fw_state - DSP firmware state definitions
+ * @SOF_FW_BOOT_NOT_STARTED:	firmware boot is not yet started
+ * @SOF_FW_BOOT_PREPARE:	preparing for boot (firmware loading for exaqmple)
+ * @SOF_FW_BOOT_IN_PROGRESS:	firmware boot is in progress
+ * @SOF_FW_BOOT_FAILED:		firmware boot failed
+ * @SOF_FW_BOOT_READY_FAILED:	firmware booted but fw_ready op failed
+ * @SOF_FW_BOOT_READY_OK:	firmware booted and fw_ready op passed
+ * @SOF_FW_BOOT_COMPLETE:	firmware is booted up and functional
+ * @SOF_FW_CRASHED:		firmware crashed after successful boot
+ */
+enum sof_fw_state {
+	SOF_FW_BOOT_NOT_STARTED = 0,
+	SOF_FW_BOOT_PREPARE,
+	SOF_FW_BOOT_IN_PROGRESS,
+	SOF_FW_BOOT_FAILED,
+	SOF_FW_BOOT_READY_FAILED,
+	SOF_FW_BOOT_READY_OK,
+	SOF_FW_BOOT_COMPLETE,
+	SOF_FW_CRASHED,
+};
+
 /*
  * SOF Platform data.
  */

sound/soc/sof/core.c

@@ -19,7 +19,7 @@
 #endif
 
 /* see SOF_DBG_ flags */
-int sof_core_debug =  IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_FIRMWARE_TRACE);
+static int sof_core_debug =  IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_FIRMWARE_TRACE);
 module_param_named(sof_debug, sof_core_debug, int, 0444);
 MODULE_PARM_DESC(sof_debug, "SOF core debug options (0x0 all off)");
 
@@ -27,6 +27,22 @@ MODULE_PARM_DESC(sof_debug, "SOF core debug options (0x0 all off)");
 #define TIMEOUT_DEFAULT_IPC_MS  500
 #define TIMEOUT_DEFAULT_BOOT_MS 2000
 
+/**
+ * sof_debug_check_flag - check if a given flag(s) is set in sof_core_debug
+ * @mask: Flag or combination of flags to check
+ *
+ * Returns true if all bits set in mask is also set in sof_core_debug, otherwise
+ * false
+ */
+bool sof_debug_check_flag(int mask)
+{
+	if ((sof_core_debug & mask) == mask)
+		return true;
+
+	return false;
+}
+EXPORT_SYMBOL(sof_debug_check_flag);
+
 /*
  * FW Panic/fault handling.
  */
@@ -52,13 +68,23 @@ static const struct sof_panic_msg panic_msg[] = {
 	{SOF_IPC_PANIC_ASSERT, "assertion failed"},
 };
 
-/*
+/**
+ * sof_print_oops_and_stack - Handle the printing of DSP oops and stack trace
+ * @sdev: Pointer to the device's sdev
+ * @level: prink log level to use for the printing
+ * @panic_code: the panic code
+ * @tracep_code: tracepoint code
+ * @oops: Pointer to DSP specific oops data
+ * @panic_info: Pointer to the received panic information message
+ * @stack: Pointer to the call stack data
+ * @stack_words: Number of words in the stack data
+ *
  * helper to be called from .dbg_dump callbacks. No error code is
  * provided, it's left as an exercise for the caller of .dbg_dump
  * (typically IPC or loader)
  */
-void snd_sof_get_status(struct snd_sof_dev *sdev, u32 panic_code,
-			u32 tracep_code, void *oops,
+void sof_print_oops_and_stack(struct snd_sof_dev *sdev, const char *level,
+			      u32 panic_code, u32 tracep_code, void *oops,
 			      struct sof_ipc_panic_info *panic_info,
 			      void *stack, size_t stack_words)
 {
@@ -67,7 +93,7 @@ void snd_sof_get_status(struct snd_sof_dev *sdev, u32 panic_code,
 
 	/* is firmware dead ? */
 	if ((panic_code & SOF_IPC_PANIC_MAGIC_MASK) != SOF_IPC_PANIC_MAGIC) {
-		dev_err(sdev->dev, "unexpected fault %#010x trace %#010x\n",
+		dev_printk(level, sdev->dev, "unexpected fault %#010x trace %#010x\n",
 			   panic_code, tracep_code);
 		return; /* no fault ? */
 	}
@@ -76,54 +102,55 @@ void snd_sof_get_status(struct snd_sof_dev *sdev, u32 panic_code,
 
 	for (i = 0; i < ARRAY_SIZE(panic_msg); i++) {
 		if (panic_msg[i].id == code) {
-			dev_err(sdev->dev, "reason: %s (%#x)\n", panic_msg[i].msg,
-				code & SOF_IPC_PANIC_CODE_MASK);
-			dev_err(sdev->dev, "trace point: %#010x\n", tracep_code);
+			dev_printk(level, sdev->dev, "reason: %s (%#x)\n",
+				   panic_msg[i].msg, code & SOF_IPC_PANIC_CODE_MASK);
+			dev_printk(level, sdev->dev, "trace point: %#010x\n", tracep_code);
 			goto out;
 		}
 	}
 
 	/* unknown error */
-	dev_err(sdev->dev, "unknown panic code: %#x\n", code & SOF_IPC_PANIC_CODE_MASK);
-	dev_err(sdev->dev, "trace point: %#010x\n", tracep_code);
+	dev_printk(level, sdev->dev, "unknown panic code: %#x\n",
+		   code & SOF_IPC_PANIC_CODE_MASK);
+	dev_printk(level, sdev->dev, "trace point: %#010x\n", tracep_code);
 
 out:
-	dev_err(sdev->dev, "panic at %s:%d\n", panic_info->filename,
+	dev_printk(level, sdev->dev, "panic at %s:%d\n", panic_info->filename,
 		   panic_info->linenum);
-	sof_oops(sdev, oops);
-	sof_stack(sdev, oops, stack, stack_words);
+	sof_oops(sdev, level, oops);
+	sof_stack(sdev, level, oops, stack, stack_words);
 }
-EXPORT_SYMBOL(snd_sof_get_status);
+EXPORT_SYMBOL(sof_print_oops_and_stack);
 
 /*
  *			FW Boot State Transition Diagram
  *
- *    +-----------------------------------------------------------------------+
+ *    +----------------------------------------------------------------------+
  *    |									     |
  * ------------------	     ------------------				     |
  * |		    |	     |		      |				     |
- * |   BOOT_FAILED  |	     |  READY_FAILED  |-------------------------+     |
- * |		    |	     |	              |				|     |
- * ------------------	     ------------------				|     |
- *	^			    ^					|     |
- *	|			    |					|     |
- * (FW Boot Timeout)		(FW_READY FAIL)				|     |
- *	|			    |					|     |
- *	|			    |					|     |
- * ------------------		    |		   ------------------	|     |
+ * |   BOOT_FAILED  |<-------|  READY_FAILED  |				     |
+ * |		    |<--+    |	              |	   ------------------	     |
+ * ------------------	|    ------------------	   |		    |	     |
+ *	^		|	    ^		   |	CRASHED	    |---+    |
  *	|		|	    |		   |		    |	|    |
+ * (FW Boot Timeout)	|	(FW_READY FAIL)	   ------------------	|    |
+ *	|		|	    |		     ^			|    |
+ *	|		|	    |		     |(DSP Panic)	|    |
+ * ------------------	|	    |		   ------------------	|    |
+ * |		    |	|	    |		   |		    |	|    |
  * |   IN_PROGRESS  |---------------+------------->|    COMPLETE    |	|    |
  * |		    | (FW Boot OK)   (FW_READY OK) |		    |	|    |
- * ------------------				   ------------------	|     |
- *	^						|		|     |
- *	|						|		|     |
- * (FW Loading OK)			       (System Suspend/Runtime Suspend)
- *	|						|		|     |
- *	|						|		|     |
- * ------------------		------------------	|		|     |
- * |		    |		|		 |<-----+		|     |
- * |   PREPARE	    |		|   NOT_STARTED  |<---------------------+     |
- * |		    |		|		 |<---------------------------+
+ * ------------------	|			   ------------------	|    |
+ *	^		|				|		|    |
+ *	|		|				|		|    |
+ * (FW Loading OK)	|			(System Suspend/Runtime Suspend)
+ *	|		|				|		|    |
+ *	|	(FW Loading Fail)			|		|    |
+ * ------------------	|	------------------	|		|    |
+ * |		    |	|	|		 |<-----+		|    |
+ * |   PREPARE	    |---+	|   NOT_STARTED  |<---------------------+    |
+ * |		    |		|		 |<--------------------------+
  * ------------------		------------------
  *    |	    ^			    |	   ^
  *    |	    |			    |	   |
@@ -186,6 +213,7 @@ static int sof_probe_continue(struct snd_sof_dev *sdev)
 	if (ret < 0) {
 		dev_err(sdev->dev, "error: failed to load DSP firmware %d\n",
 			ret);
+		sof_set_fw_state(sdev, SOF_FW_BOOT_FAILED);
 		goto fw_load_err;
 	}
 
@@ -199,10 +227,11 @@ static int sof_probe_continue(struct snd_sof_dev *sdev)
 	if (ret < 0) {
 		dev_err(sdev->dev, "error: failed to boot DSP firmware %d\n",
 			ret);
+		sof_set_fw_state(sdev, SOF_FW_BOOT_FAILED);
 		goto fw_run_err;
 	}
 
-	if (sof_core_debug & SOF_DBG_ENABLE_TRACE) {
+	if (sof_debug_check_flag(SOF_DBG_ENABLE_TRACE)) {
 		sdev->dtrace_is_supported = true;
 
 		/* init DMA trace */

sound/soc/sof/debug.c

@@ -930,7 +930,7 @@ void snd_sof_free_debug(struct snd_sof_dev *sdev)
 EXPORT_SYMBOL_GPL(snd_sof_free_debug);
 
 static const struct soc_fw_state_info {
-	enum snd_sof_fw_state state;
+	enum sof_fw_state state;
 	const char *name;
 } fw_state_dbg[] = {
 	{SOF_FW_BOOT_NOT_STARTED, "SOF_FW_BOOT_NOT_STARTED"},
@@ -938,37 +938,47 @@ static const struct soc_fw_state_info {
 	{SOF_FW_BOOT_IN_PROGRESS, "SOF_FW_BOOT_IN_PROGRESS"},
 	{SOF_FW_BOOT_FAILED, "SOF_FW_BOOT_FAILED"},
 	{SOF_FW_BOOT_READY_FAILED, "SOF_FW_BOOT_READY_FAILED"},
+	{SOF_FW_BOOT_READY_OK, "SOF_FW_BOOT_READY_OK"},
 	{SOF_FW_BOOT_COMPLETE, "SOF_FW_BOOT_COMPLETE"},
+	{SOF_FW_CRASHED, "SOF_FW_CRASHED"},
 };
 
-static void snd_sof_dbg_print_fw_state(struct snd_sof_dev *sdev)
+static void snd_sof_dbg_print_fw_state(struct snd_sof_dev *sdev, const char *level)
 {
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(fw_state_dbg); i++) {
 		if (sdev->fw_state == fw_state_dbg[i].state) {
-			dev_err(sdev->dev, "fw_state: %s (%d)\n", fw_state_dbg[i].name, i);
+			dev_printk(level, sdev->dev, "fw_state: %s (%d)\n",
+				   fw_state_dbg[i].name, i);
 			return;
 		}
 	}
 
-	dev_err(sdev->dev, "fw_state: UNKNOWN (%d)\n", sdev->fw_state);
+	dev_printk(level, sdev->dev, "fw_state: UNKNOWN (%d)\n", sdev->fw_state);
 }
 
-void snd_sof_dsp_dbg_dump(struct snd_sof_dev *sdev, u32 flags)
+void snd_sof_dsp_dbg_dump(struct snd_sof_dev *sdev, const char *msg, u32 flags)
 {
-	bool print_all = !!(sof_core_debug & SOF_DBG_PRINT_ALL_DUMPS);
+	char *level = flags & SOF_DBG_DUMP_OPTIONAL ? KERN_DEBUG : KERN_ERR;
+	bool print_all = sof_debug_check_flag(SOF_DBG_PRINT_ALL_DUMPS);
 
 	if (flags & SOF_DBG_DUMP_OPTIONAL && !print_all)
 		return;
 
 	if (sof_ops(sdev)->dbg_dump && !sdev->dbg_dump_printed) {
-		dev_err(sdev->dev, "------------[ DSP dump start ]------------\n");
-		snd_sof_dbg_print_fw_state(sdev);
+		dev_printk(level, sdev->dev,
+			   "------------[ DSP dump start ]------------\n");
+		if (msg)
+			dev_printk(level, sdev->dev, "%s\n", msg);
+		snd_sof_dbg_print_fw_state(sdev, level);
 		sof_ops(sdev)->dbg_dump(sdev, flags);
-		dev_err(sdev->dev, "------------[ DSP dump end ]------------\n");
+		dev_printk(level, sdev->dev,
+			   "------------[ DSP dump end ]------------\n");
 		if (!print_all)
 			sdev->dbg_dump_printed = true;
+	} else if (msg) {
+		dev_printk(level, sdev->dev, "%s\n", msg);
 	}
 }
 EXPORT_SYMBOL(snd_sof_dsp_dbg_dump);
@@ -979,7 +989,7 @@ static void snd_sof_ipc_dump(struct snd_sof_dev *sdev)
 		dev_err(sdev->dev, "------------[ IPC dump start ]------------\n");
 		sof_ops(sdev)->ipc_dump(sdev);
 		dev_err(sdev->dev, "------------[ IPC dump end ]------------\n");
-		if (!(sof_core_debug & SOF_DBG_PRINT_ALL_DUMPS))
+		if (!sof_debug_check_flag(SOF_DBG_PRINT_ALL_DUMPS))
 			sdev->ipc_dump_printed = true;
 	}
 }
@@ -987,7 +997,7 @@ static void snd_sof_ipc_dump(struct snd_sof_dev *sdev)
 void snd_sof_handle_fw_exception(struct snd_sof_dev *sdev)
 {
 	if (IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_RETAIN_DSP_CONTEXT) ||
-	    (sof_core_debug & SOF_DBG_RETAIN_CTX)) {
+	    sof_debug_check_flag(SOF_DBG_RETAIN_CTX)) {
 		/* should we prevent DSP entering D3 ? */
 		if (!sdev->ipc_dump_printed)
 			dev_info(sdev->dev,
@@ -997,7 +1007,8 @@ void snd_sof_handle_fw_exception(struct snd_sof_dev *sdev)
 
 	/* dump vital information to the logs */
 	snd_sof_ipc_dump(sdev);
-	snd_sof_dsp_dbg_dump(sdev, SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX);
+	snd_sof_dsp_dbg_dump(sdev, "Firmware exception",
+			     SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX);
 	snd_sof_trace_notify_for_error(sdev);
 }
 EXPORT_SYMBOL(snd_sof_handle_fw_exception);

sound/soc/sof/imx/imx-common.c

@@ -69,8 +69,8 @@ void imx8_dump(struct snd_sof_dev *sdev, u32 flags)
 			   IMX8_STACK_DUMP_SIZE);
 
 	/* Print the information to the console */
-	snd_sof_get_status(sdev, status, status, &xoops, &panic_info, stack,
-			   IMX8_STACK_DUMP_SIZE);
+	sof_print_oops_and_stack(sdev, KERN_ERR, status, status, &xoops,
+				 &panic_info, stack, IMX8_STACK_DUMP_SIZE);
 }
 EXPORT_SYMBOL(imx8_dump);
 

sound/soc/sof/imx/imx8.c

@@ -97,7 +97,7 @@ static void imx8_dsp_handle_request(struct imx_dsp_ipc *ipc)
 
 	/* Check to see if the message is a panic code (0x0dead***) */
 	if ((p & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC)
-		snd_sof_dsp_panic(priv->sdev, p);
+		snd_sof_dsp_panic(priv->sdev, p, true);
 	else
 		snd_sof_ipc_msgs_rx(priv->sdev);
 }

sound/soc/sof/imx/imx8m.c

@@ -90,7 +90,7 @@ static void imx8m_dsp_handle_request(struct imx_dsp_ipc *ipc)
 
 	/* Check to see if the message is a panic code (0x0dead***) */
 	if ((p & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC)
-		snd_sof_dsp_panic(priv->sdev, p);
+		snd_sof_dsp_panic(priv->sdev, p, true);
 	else
 		snd_sof_ipc_msgs_rx(priv->sdev);
 }

sound/soc/sof/intel/atom.c

@@ -70,8 +70,8 @@ void atom_dump(struct snd_sof_dev *sdev, u32 flags)
 	panic = snd_sof_dsp_read64(sdev, DSP_BAR, SHIM_IPCX);
 	atom_get_registers(sdev, &xoops, &panic_info, stack,
 			   STACK_DUMP_SIZE);
-	snd_sof_get_status(sdev, status, panic, &xoops, &panic_info, stack,
-			   STACK_DUMP_SIZE);
+	sof_print_oops_and_stack(sdev, KERN_ERR, status, panic, &xoops,
+				 &panic_info, stack, STACK_DUMP_SIZE);
 
 	/* provide some context for firmware debug */
 	imrx = snd_sof_dsp_read64(sdev, DSP_BAR, SHIM_IMRX);
@@ -165,8 +165,8 @@ irqreturn_t atom_irq_thread(int irq, void *context)
 
 		/* Handle messages from DSP Core */
 		if ((ipcd & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) {
-			snd_sof_dsp_panic(sdev, PANIC_OFFSET(ipcd) +
-					  MBOX_OFFSET);
+			snd_sof_dsp_panic(sdev, PANIC_OFFSET(ipcd) + MBOX_OFFSET,
+					  true);
 		} else {
 			snd_sof_ipc_msgs_rx(sdev);
 		}

sound/soc/sof/intel/bdw.c

@@ -258,8 +258,8 @@ static void bdw_dump(struct snd_sof_dev *sdev, u32 flags)
 	panic = snd_sof_dsp_read(sdev, BDW_DSP_BAR, SHIM_IPCX);
 	bdw_get_registers(sdev, &xoops, &panic_info, stack,
 			  BDW_STACK_DUMP_SIZE);
-	snd_sof_get_status(sdev, status, panic, &xoops, &panic_info, stack,
-			   BDW_STACK_DUMP_SIZE);
+	sof_print_oops_and_stack(sdev, KERN_ERR, status, panic, &xoops,
+				 &panic_info, stack, BDW_STACK_DUMP_SIZE);
 
 	/* provide some context for firmware debug */
 	imrx = snd_sof_dsp_read(sdev, BDW_DSP_BAR, SHIM_IMRX);
@@ -344,8 +344,8 @@ static irqreturn_t bdw_irq_thread(int irq, void *context)
 
 		/* Handle messages from DSP Core */
 		if ((ipcd & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) {
-			snd_sof_dsp_panic(sdev, BDW_PANIC_OFFSET(ipcx) +
-					  MBOX_OFFSET);
+			snd_sof_dsp_panic(sdev, BDW_PANIC_OFFSET(ipcx) + MBOX_OFFSET,
+					  true);
 		} else {
 			snd_sof_ipc_msgs_rx(sdev);
 		}

sound/soc/sof/intel/cnl.c

@@ -82,9 +82,24 @@ irqreturn_t cnl_ipc_irq_thread(int irq, void *context)
 			 msg, msg_ext);
 
 		/* handle messages from DSP */
-		if ((hipctdr & SOF_IPC_PANIC_MAGIC_MASK) ==
-		   SOF_IPC_PANIC_MAGIC) {
-			snd_sof_dsp_panic(sdev, HDA_DSP_PANIC_OFFSET(msg_ext));
+		if ((hipctdr & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) {
+			struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+			bool non_recoverable = true;
+
+			/*
+			 * This is a PANIC message!
+			 *
+			 * If it is arriving during firmware boot and it is not
+			 * the last boot attempt then change the non_recoverable
+			 * to false as the DSP might be able to boot in the next
+			 * iteration(s)
+			 */
+			if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS &&
+			    hda->boot_iteration < HDA_FW_BOOT_ATTEMPTS)
+				non_recoverable = false;
+
+			snd_sof_dsp_panic(sdev, HDA_DSP_PANIC_OFFSET(msg_ext),
+					  non_recoverable);
 		} else {
 			snd_sof_ipc_msgs_rx(sdev);
 		}

sound/soc/sof/intel/hda-ipc.c

@@ -173,8 +173,23 @@ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context)
 
 		/* handle messages from DSP */
 		if ((hipct & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) {
-			/* this is a PANIC message !! */
-			snd_sof_dsp_panic(sdev, HDA_DSP_PANIC_OFFSET(msg_ext));
+			struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+			bool non_recoverable = true;
+
+			/*
+			 * This is a PANIC message!
+			 *
+			 * If it is arriving during firmware boot and it is not
+			 * the last boot attempt then change the non_recoverable
+			 * to false as the DSP might be able to boot in the next
+			 * iteration(s)
+			 */
+			if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS &&
+			    hda->boot_iteration < HDA_FW_BOOT_ATTEMPTS)
+				non_recoverable = false;
+
+			snd_sof_dsp_panic(sdev, HDA_DSP_PANIC_OFFSET(msg_ext),
+					  non_recoverable);
 		} else {
 			/* normal message - process normally */
 			snd_sof_ipc_msgs_rx(sdev);

sound/soc/sof/intel/hda-loader.c

@@ -23,7 +23,6 @@
 #include "../ops.h"
 #include "hda.h"
 
-#define HDA_FW_BOOT_ATTEMPTS	3
 #define HDA_CL_STREAM_FORMAT 0x40
 
 static struct hdac_ext_stream *cl_stream_prepare(struct snd_sof_dev *sdev, unsigned int format,
@@ -89,6 +88,7 @@ static int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag)
 	const struct sof_intel_dsp_desc *chip = hda->desc;
 	unsigned int status;
 	unsigned long mask;
+	char *dump_msg;
 	u32 flags, j;
 	int ret;
 	int i;
@@ -190,9 +190,12 @@ static int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag)
 	if (hda->boot_iteration == HDA_FW_BOOT_ATTEMPTS)
 		flags &= ~SOF_DBG_DUMP_OPTIONAL;
 
-	snd_sof_dsp_dbg_dump(sdev, flags);
+	dump_msg = kasprintf(GFP_KERNEL, "Boot iteration failed: %d/%d",
+			     hda->boot_iteration, HDA_FW_BOOT_ATTEMPTS);
+	snd_sof_dsp_dbg_dump(sdev, dump_msg, flags);
 	hda_dsp_core_reset_power_down(sdev, chip->host_managed_cores_mask);
 
+	kfree(dump_msg);
 	return ret;
 }
 
@@ -414,16 +417,19 @@ int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev)
 		hda_sdw_process_wakeen(sdev);
 
 	/*
-	 * at this point DSP ROM has been initialized and
-	 * should be ready for code loading and firmware boot
+	 * Set the boot_iteration to the last attempt, indicating that the
+	 * DSP ROM has been initialized and from this point there will be no
+	 * retry done to boot.
+	 *
+	 * Continue with code loading and firmware boot
 	 */
+	hda->boot_iteration = HDA_FW_BOOT_ATTEMPTS;
 	ret = cl_copy_fw(sdev, stream);
-	if (!ret) {
+	if (!ret)
 		dev_dbg(sdev->dev, "Firmware download successful, booting...\n");
-	} else {
-		snd_sof_dsp_dbg_dump(sdev, SOF_DBG_DUMP_PCI | SOF_DBG_DUMP_MBOX);
-		dev_err(sdev->dev, "error: load fw failed ret: %d\n", ret);
-	}
+	else
+		snd_sof_dsp_dbg_dump(sdev, "Firmware download failed",
+				     SOF_DBG_DUMP_PCI | SOF_DBG_DUMP_MBOX);
 
 cleanup:
 	/*

sound/soc/sof/intel/hda.c

@@ -474,7 +474,7 @@ static const struct hda_dsp_msg_code hda_dsp_rom_msg[] = {
 	{HDA_DSP_ROM_NULL_FW_ENTRY,	"error: null FW entry point"},
 };
 
-static void hda_dsp_get_status(struct snd_sof_dev *sdev)
+static void hda_dsp_get_status(struct snd_sof_dev *sdev, const char *level)
 {
 	u32 status;
 	int i;
@@ -484,7 +484,7 @@ static void hda_dsp_get_status(struct snd_sof_dev *sdev)
 
 	for (i = 0; i < ARRAY_SIZE(hda_dsp_rom_msg); i++) {
 		if (status == hda_dsp_rom_msg[i].code) {
-			dev_err(sdev->dev, "%s - code %8.8x\n",
+			dev_printk(level, sdev->dev, "%s - code %8.8x\n",
 				   hda_dsp_rom_msg[i].msg, status);
 			return;
 		}
@@ -523,7 +523,8 @@ static void hda_dsp_get_registers(struct snd_sof_dev *sdev,
 }
 
 /* dump the first 8 dwords representing the extended ROM status */
-static void hda_dsp_dump_ext_rom_status(struct snd_sof_dev *sdev, u32 flags)
+static void hda_dsp_dump_ext_rom_status(struct snd_sof_dev *sdev, const char *level,
+					u32 flags)
 {
 	char msg[128];
 	int len = 0;
@@ -535,18 +536,19 @@ static void hda_dsp_dump_ext_rom_status(struct snd_sof_dev *sdev, u32 flags)
 		len += snprintf(msg + len, sizeof(msg) - len, " 0x%x", value);
 	}
 
-	dev_err(sdev->dev, "extended rom status: %s", msg);
+	dev_printk(level, sdev->dev, "extended rom status: %s", msg);
 
 }
 
 void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags)
 {
+	char *level = flags & SOF_DBG_DUMP_OPTIONAL ? KERN_DEBUG : KERN_ERR;
 	struct sof_ipc_dsp_oops_xtensa xoops;
 	struct sof_ipc_panic_info panic_info;
 	u32 stack[HDA_DSP_STACK_DUMP_SIZE];
 
 	/* print ROM/FW status */
-	hda_dsp_get_status(sdev);
+	hda_dsp_get_status(sdev, level);
 
 	if (flags & SOF_DBG_DUMP_REGS) {
 		u32 status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_FW_STATUS);
@@ -554,10 +556,10 @@ void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags)
 
 		hda_dsp_get_registers(sdev, &xoops, &panic_info, stack,
 				      HDA_DSP_STACK_DUMP_SIZE);
-		snd_sof_get_status(sdev, status, panic, &xoops, &panic_info,
-				   stack, HDA_DSP_STACK_DUMP_SIZE);
+		sof_print_oops_and_stack(sdev, level, status, panic, &xoops,
+					 &panic_info, stack, HDA_DSP_STACK_DUMP_SIZE);
 	} else {
-		hda_dsp_dump_ext_rom_status(sdev, flags);
+		hda_dsp_dump_ext_rom_status(sdev, level, flags);
 	}
 }
 

sound/soc/sof/ipc.c

@@ -302,7 +302,7 @@ static int sof_ipc_tx_message_unlocked(struct snd_sof_ipc *ipc, u32 header,
 	struct snd_sof_ipc_msg *msg;
 	int ret;
 
-	if (ipc->disable_ipc_tx)
+	if (ipc->disable_ipc_tx || sdev->fw_state != SOF_FW_BOOT_COMPLETE)
 		return -ENODEV;
 
 	/*
@@ -536,7 +536,7 @@ void snd_sof_ipc_msgs_rx(struct snd_sof_dev *sdev)
 			if (err < 0)
 				sof_set_fw_state(sdev, SOF_FW_BOOT_READY_FAILED);
 			else
-				sof_set_fw_state(sdev, SOF_FW_BOOT_COMPLETE);
+				sof_set_fw_state(sdev, SOF_FW_BOOT_READY_OK);
 
 			/* wake up firmware loader */
 			wake_up(&sdev->boot_wait);

sound/soc/sof/loader.c

@@ -820,8 +820,8 @@ int snd_sof_run_firmware(struct snd_sof_dev *sdev)
 	/* boot the firmware on the DSP */
 	ret = snd_sof_dsp_run(sdev);
 	if (ret < 0) {
-		dev_err(sdev->dev, "error: failed to start DSP\n");
-		snd_sof_dsp_dbg_dump(sdev, SOF_DBG_DUMP_MBOX | SOF_DBG_DUMP_PCI);
+		snd_sof_dsp_dbg_dump(sdev, "Failed to start DSP",
+				     SOF_DBG_DUMP_MBOX | SOF_DBG_DUMP_PCI);
 		return ret;
 	}
 
@@ -835,16 +835,13 @@ int snd_sof_run_firmware(struct snd_sof_dev *sdev)
 				 sdev->fw_state > SOF_FW_BOOT_IN_PROGRESS,
 				 msecs_to_jiffies(sdev->boot_timeout));
 	if (ret == 0) {
-		dev_err(sdev->dev, "error: firmware boot failure\n");
-		snd_sof_dsp_dbg_dump(sdev, SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX |
+		snd_sof_dsp_dbg_dump(sdev, "Firmware boot failure due to timeout",
+				     SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX |
 				     SOF_DBG_DUMP_TEXT | SOF_DBG_DUMP_PCI);
-		sof_set_fw_state(sdev, SOF_FW_BOOT_FAILED);
 		return -EIO;
 	}
 
-	if (sdev->fw_state == SOF_FW_BOOT_COMPLETE)
-		dev_dbg(sdev->dev, "firmware boot complete\n");
-	else
+	if (sdev->fw_state == SOF_FW_BOOT_READY_FAILED)
 		return -EIO; /* FW boots but fw_ready op failed */
 
 	/* perform post fw run operations */
@@ -854,6 +851,9 @@ int snd_sof_run_firmware(struct snd_sof_dev *sdev)
 		return ret;
 	}
 
+	dev_dbg(sdev->dev, "firmware boot complete\n");
+	sof_set_fw_state(sdev, SOF_FW_BOOT_COMPLETE);
+
 	return 0;
 }
 EXPORT_SYMBOL(snd_sof_run_firmware);

sound/soc/sof/ops.c

@@ -142,25 +142,46 @@ void snd_sof_dsp_update_bits_forced(struct snd_sof_dev *sdev, u32 bar,
 }
 EXPORT_SYMBOL(snd_sof_dsp_update_bits_forced);
 
-void snd_sof_dsp_panic(struct snd_sof_dev *sdev, u32 offset)
+/**
+ * snd_sof_dsp_panic - handle a received DSP panic message
+ * @sdev: Pointer to the device's sdev
+ * @offset: offset of panic information
+ * @non_recoverable: the panic is fatal, no recovery will be done by the caller
+ */
+void snd_sof_dsp_panic(struct snd_sof_dev *sdev, u32 offset, bool non_recoverable)
 {
-	dev_err(sdev->dev, "error : DSP panic!\n");
-
 	/*
-	 * check if DSP is not ready and did not set the dsp_oops_offset.
-	 * if the dsp_oops_offset is not set, set it from the panic message.
-	 * Also add a check to memory window setting with panic message.
+	 * if DSP is not ready and the dsp_oops_offset is not yet set, use the
+	 * offset from the panic message.
 	 */
 	if (!sdev->dsp_oops_offset)
 		sdev->dsp_oops_offset = offset;
-	else
-		dev_dbg(sdev->dev, "panic: dsp_oops_offset %zu offset %d\n",
-			sdev->dsp_oops_offset, offset);
 
-	/* We want to see the DSP panic! */
-	sdev->dbg_dump_printed = false;
+	/*
+	 * Print warning if the offset from the panic message differs from
+	 * dsp_oops_offset
+	 */
+	if (sdev->dsp_oops_offset != offset)
+		dev_warn(sdev->dev,
+			 "%s: dsp_oops_offset %zu differs from panic offset %u\n",
+			 __func__, sdev->dsp_oops_offset, offset);
 
-	snd_sof_dsp_dbg_dump(sdev, SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX);
+	/*
+	 * Set the fw_state to crashed only in case of non recoverable DSP panic
+	 * event.
+	 * Use different message within the snd_sof_dsp_dbg_dump() depending on
+	 * the non_recoverable flag.
+	 */
+	sdev->dbg_dump_printed = false;
+	if (non_recoverable) {
+		snd_sof_dsp_dbg_dump(sdev, "DSP panic!",
+				     SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX);
+		sof_set_fw_state(sdev, SOF_FW_CRASHED);
 		snd_sof_trace_notify_for_error(sdev);
+	} else {
+		snd_sof_dsp_dbg_dump(sdev,
+				     "DSP panic (recovery will be attempted)",
+				     SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX);
+	}
 }
 EXPORT_SYMBOL(snd_sof_dsp_panic);

sound/soc/sof/ops.h

@@ -274,7 +274,7 @@ snd_sof_dsp_set_power_state(struct snd_sof_dev *sdev,
 }
 
 /* debug */
-void snd_sof_dsp_dbg_dump(struct snd_sof_dev *sdev, u32 flags);
+void snd_sof_dsp_dbg_dump(struct snd_sof_dev *sdev, const char *msg, u32 flags);
 
 static inline int snd_sof_debugfs_add_region_item(struct snd_sof_dev *sdev,
 		enum snd_sof_fw_blk_type blk_type, u32 offset, size_t size,
@@ -643,5 +643,5 @@ int snd_sof_dsp_register_poll(struct snd_sof_dev *sdev, u32 bar, u32 offset,
 			      u32 mask, u32 target, u32 timeout_ms,
 			      u32 interval_us);
 
-void snd_sof_dsp_panic(struct snd_sof_dev *sdev, u32 offset);
+void snd_sof_dsp_panic(struct snd_sof_dev *sdev, u32 offset, bool non_recoverable);
 #endif

sound/soc/sof/pm.c

@@ -130,6 +130,7 @@ static int sof_resume(struct device *dev, bool runtime_resume)
 		dev_err(sdev->dev,
 			"error: failed to load DSP firmware after resume %d\n",
 			ret);
+		sof_set_fw_state(sdev, SOF_FW_BOOT_FAILED);
 		return ret;
 	}
 
@@ -144,6 +145,7 @@ static int sof_resume(struct device *dev, bool runtime_resume)
 		dev_err(sdev->dev,
 			"error: failed to boot DSP firmware after resume %d\n",
 			ret);
+		sof_set_fw_state(sdev, SOF_FW_BOOT_FAILED);
 		return ret;
 	}
 
@@ -312,6 +314,14 @@ int snd_sof_prepare(struct device *dev)
 	/* will suspend to S3 by default */
 	sdev->system_suspend_target = SOF_SUSPEND_S3;
 
+	/*
+	 * if the firmware is crashed or boot failed then we try to aim for S3
+	 * to reboot the firmware
+	 */
+	if (sdev->fw_state == SOF_FW_CRASHED ||
+	    sdev->fw_state == SOF_FW_BOOT_FAILED)
+		return 0;
+
 	if (!desc->use_acpi_target_states)
 		return 0;
 

sound/soc/sof/sof-priv.h

@@ -20,7 +20,7 @@
 #include <uapi/sound/sof/fw.h>
 #include <sound/sof/ext_manifest.h>
 
-/* debug flags */
+/* Flag definitions used in sof_core_debug (sof_debug module parameter) */
 #define SOF_DBG_ENABLE_TRACE	BIT(0)
 #define SOF_DBG_RETAIN_CTX	BIT(1)	/* prevent DSP D3 on FW exception */
 #define SOF_DBG_VERIFY_TPLG	BIT(2) /* verify topology during load */
@@ -35,14 +35,16 @@
 							*/
 #define SOF_DBG_PRINT_ALL_DUMPS		BIT(6) /* Print all ipc and dsp dumps */
 
+/* Flag definitions used for controlling the DSP dump behavior */
 #define SOF_DBG_DUMP_REGS		BIT(0)
 #define SOF_DBG_DUMP_MBOX		BIT(1)
 #define SOF_DBG_DUMP_TEXT		BIT(2)
 #define SOF_DBG_DUMP_PCI		BIT(3)
-#define SOF_DBG_DUMP_OPTIONAL		BIT(4) /* only dump if SOF_DBG_PRINT_ALL_DUMPS is set */
+/* Output this dump (at the DEBUG level) only when SOF_DBG_PRINT_ALL_DUMPS is set */
+#define SOF_DBG_DUMP_OPTIONAL		BIT(4)
 
 /* global debug state set by SOF_DBG_ flags */
-extern int sof_core_debug;
+bool sof_debug_check_flag(int mask);
 
 /* max BARs mmaped devices can use */
 #define SND_SOF_BARS	8
@@ -309,8 +311,8 @@ struct snd_sof_dsp_ops {
 
 /* DSP architecture specific callbacks for oops and stack dumps */
 struct dsp_arch_ops {
-	void (*dsp_oops)(struct snd_sof_dev *sdev, void *oops);
-	void (*dsp_stack)(struct snd_sof_dev *sdev, void *oops,
+	void (*dsp_oops)(struct snd_sof_dev *sdev, const char *level, void *oops);
+	void (*dsp_stack)(struct snd_sof_dev *sdev, const char *level, void *oops,
 			  u32 *stack, u32 stack_words);
 };
 
@@ -375,15 +377,6 @@ struct snd_sof_ipc_msg {
 	bool ipc_complete;
 };
 
-enum snd_sof_fw_state {
-	SOF_FW_BOOT_NOT_STARTED = 0,
-	SOF_FW_BOOT_PREPARE,
-	SOF_FW_BOOT_IN_PROGRESS,
-	SOF_FW_BOOT_FAILED,
-	SOF_FW_BOOT_READY_FAILED, /* firmware booted but fw_ready op failed */
-	SOF_FW_BOOT_COMPLETE,
-};
-
 /*
  * SOF Device Level.
  */
@@ -408,7 +401,7 @@ struct snd_sof_dev {
 
 	/* DSP firmware boot */
 	wait_queue_head_t boot_wait;
-	enum snd_sof_fw_state fw_state;
+	enum sof_fw_state fw_state;
 	bool first_boot;
 
 	/* work queue in case the probe is implemented in two steps */
@@ -568,8 +561,8 @@ int snd_sof_debugfs_buf_item(struct snd_sof_dev *sdev,
 int snd_sof_trace_update_pos(struct snd_sof_dev *sdev,
 			     struct sof_ipc_dma_trace_posn *posn);
 void snd_sof_trace_notify_for_error(struct snd_sof_dev *sdev);
-void snd_sof_get_status(struct snd_sof_dev *sdev, u32 panic_code,
-			u32 tracep_code, void *oops,
+void sof_print_oops_and_stack(struct snd_sof_dev *sdev, const char *level,
+			      u32 panic_code, u32 tracep_code, void *oops,
 			      struct sof_ipc_panic_info *panic_info,
 			      void *stack, size_t stack_words);
 int snd_sof_init_trace_ipc(struct snd_sof_dev *sdev);
@@ -582,16 +575,17 @@ int snd_sof_debugfs_add_region_item_iomem(struct snd_sof_dev *sdev,
 /*
  * DSP Architectures.
  */
-static inline void sof_stack(struct snd_sof_dev *sdev, void *oops, u32 *stack,
-			     u32 stack_words)
+static inline void sof_stack(struct snd_sof_dev *sdev, const char *level,
+			     void *oops, u32 *stack, u32 stack_words)
 {
-		sof_dsp_arch_ops(sdev)->dsp_stack(sdev, oops, stack, stack_words);
+		sof_dsp_arch_ops(sdev)->dsp_stack(sdev, level,  oops, stack,
+						  stack_words);
 }
 
-static inline void sof_oops(struct snd_sof_dev *sdev, void *oops)
+static inline void sof_oops(struct snd_sof_dev *sdev, const char *level, void *oops)
 {
 	if (sof_dsp_arch_ops(sdev)->dsp_oops)
-		sof_dsp_arch_ops(sdev)->dsp_oops(sdev, oops);
+		sof_dsp_arch_ops(sdev)->dsp_oops(sdev, level, oops);
 }
 
 extern const struct dsp_arch_ops sof_xtensa_arch_ops;
@@ -600,7 +594,7 @@ extern const struct dsp_arch_ops sof_xtensa_arch_ops;
  * Firmware state tracking
  */
 static inline void sof_set_fw_state(struct snd_sof_dev *sdev,
-				    enum snd_sof_fw_state new_state)
+				    enum sof_fw_state new_state)
 {
 	if (sdev->fw_state == new_state)
 		return;

sound/soc/sof/topology.c

@@ -1695,12 +1695,12 @@ static int sof_widget_load_pipeline(struct snd_soc_component *scomp, int index,
 		goto err;
 	}
 
-	if (sof_core_debug & SOF_DBG_DISABLE_MULTICORE)
+	if (sof_debug_check_flag(SOF_DBG_DISABLE_MULTICORE))
 		pipeline->core = SOF_DSP_PRIMARY_CORE;
 
-	if (sof_core_debug & SOF_DBG_DYNAMIC_PIPELINES_OVERRIDE)
-		swidget->dynamic_pipeline_widget = sof_core_debug &
-			SOF_DBG_DYNAMIC_PIPELINES_ENABLE;
+	if (sof_debug_check_flag(SOF_DBG_DYNAMIC_PIPELINES_OVERRIDE))
+		swidget->dynamic_pipeline_widget =
+			sof_debug_check_flag(SOF_DBG_DYNAMIC_PIPELINES_ENABLE);
 
 	dev_dbg(scomp->dev, "pipeline %s: period %d pri %d mips %d core %d frames %d dynamic %d\n",
 		swidget->widget->name, pipeline->period, pipeline->priority,
@@ -2295,7 +2295,7 @@ static int sof_widget_ready(struct snd_soc_component *scomp, int index,
 		return ret;
 	}
 
-	if (sof_core_debug & SOF_DBG_DISABLE_MULTICORE)
+	if (sof_debug_check_flag(SOF_DBG_DISABLE_MULTICORE))
 		comp.core = SOF_DSP_PRIMARY_CORE;
 
 	swidget->core = comp.core;
@@ -3542,7 +3542,7 @@ static int sof_complete(struct snd_soc_component *scomp)
 	}
 
 	/* verify topology components loading including dynamic pipelines */
-	if (sof_core_debug & SOF_DBG_VERIFY_TPLG) {
+	if (sof_debug_check_flag(SOF_DBG_VERIFY_TPLG)) {
 		ret = sof_set_up_pipelines(sdev, true);
 		if (ret < 0) {
 			dev_err(sdev->dev, "error: topology verification failed %d\n", ret);

sound/soc/sof/xtensa/core.c

@@ -81,33 +81,39 @@ static const struct xtensa_exception_cause xtensa_exception_causes[] = {
 };
 
 /* only need xtensa atm */
-static void xtensa_dsp_oops(struct snd_sof_dev *sdev, void *oops)
+static void xtensa_dsp_oops(struct snd_sof_dev *sdev, const char *level, void *oops)
 {
 	struct sof_ipc_dsp_oops_xtensa *xoops = oops;
 	int i;
 
-	dev_err(sdev->dev, "error: DSP Firmware Oops\n");
+	dev_printk(level, sdev->dev, "error: DSP Firmware Oops\n");
 	for (i = 0; i < ARRAY_SIZE(xtensa_exception_causes); i++) {
 		if (xtensa_exception_causes[i].id == xoops->exccause) {
-			dev_err(sdev->dev, "error: Exception Cause: %s, %s\n",
+			dev_printk(level, sdev->dev,
+				   "error: Exception Cause: %s, %s\n",
 				   xtensa_exception_causes[i].msg,
 				   xtensa_exception_causes[i].description);
 		}
 	}
-	dev_err(sdev->dev, "EXCCAUSE 0x%8.8x EXCVADDR 0x%8.8x PS       0x%8.8x SAR     0x%8.8x\n",
+	dev_printk(level, sdev->dev,
+		   "EXCCAUSE 0x%8.8x EXCVADDR 0x%8.8x PS       0x%8.8x SAR     0x%8.8x\n",
 		   xoops->exccause, xoops->excvaddr, xoops->ps, xoops->sar);
-	dev_err(sdev->dev, "EPC1     0x%8.8x EPC2     0x%8.8x EPC3     0x%8.8x EPC4    0x%8.8x",
+	dev_printk(level, sdev->dev,
+		   "EPC1     0x%8.8x EPC2     0x%8.8x EPC3     0x%8.8x EPC4    0x%8.8x",
 		   xoops->epc1, xoops->epc2, xoops->epc3, xoops->epc4);
-	dev_err(sdev->dev, "EPC5     0x%8.8x EPC6     0x%8.8x EPC7     0x%8.8x DEPC    0x%8.8x",
+	dev_printk(level, sdev->dev,
+		   "EPC5     0x%8.8x EPC6     0x%8.8x EPC7     0x%8.8x DEPC    0x%8.8x",
 		   xoops->epc5, xoops->epc6, xoops->epc7, xoops->depc);
-	dev_err(sdev->dev, "EPS2     0x%8.8x EPS3     0x%8.8x EPS4     0x%8.8x EPS5    0x%8.8x",
+	dev_printk(level, sdev->dev,
+		   "EPS2     0x%8.8x EPS3     0x%8.8x EPS4     0x%8.8x EPS5    0x%8.8x",
 		   xoops->eps2, xoops->eps3, xoops->eps4, xoops->eps5);
-	dev_err(sdev->dev, "EPS6     0x%8.8x EPS7     0x%8.8x INTENABL 0x%8.8x INTERRU 0x%8.8x",
+	dev_printk(level, sdev->dev,
+		   "EPS6     0x%8.8x EPS7     0x%8.8x INTENABL 0x%8.8x INTERRU 0x%8.8x",
 		   xoops->eps6, xoops->eps7, xoops->intenable, xoops->interrupt);
 }
 
-static void xtensa_stack(struct snd_sof_dev *sdev, void *oops, u32 *stack,
-			 u32 stack_words)
+static void xtensa_stack(struct snd_sof_dev *sdev, const char *level, void *oops,
+			 u32 *stack, u32 stack_words)
 {
 	struct sof_ipc_dsp_oops_xtensa *xoops = oops;
 	u32 stack_ptr = xoops->plat_hdr.stackptr;
@@ -115,7 +121,7 @@ static void xtensa_stack(struct snd_sof_dev *sdev, void *oops, u32 *stack,
 	unsigned char buf[4 * 8 + 3 + 1];
 	int i;
 
-	dev_err(sdev->dev, "stack dump from 0x%8.8x\n", stack_ptr);
+	dev_printk(level, sdev->dev, "stack dump from 0x%8.8x\n", stack_ptr);
 
 	/*
 	 * example output:
@@ -124,7 +130,7 @@ static void xtensa_stack(struct snd_sof_dev *sdev, void *oops, u32 *stack,
 	for (i = 0; i < stack_words; i += 4) {
 		hex_dump_to_buffer(stack + i, 16, 16, 4,
 				   buf, sizeof(buf), false);
-		dev_err(sdev->dev, "0x%08x: %s\n", stack_ptr + i * 4, buf);
+		dev_printk(level, sdev->dev, "0x%08x: %s\n", stack_ptr + i * 4, buf);
 	}
 }