powerpc/powernv: Return secondary CPUs to firmware on kexec

With OPAL v3 we can return secondary CPUs to firmware on kexec. This
allows firmware to do various cleanups making things generally more
reliable, and will enable the "new" kernel to call OPAL to perform
some reconfiguration tasks early on that can only be done while
all the CPUs are in firmware.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

arch/powerpc/include/asm/opal.h

@@ -128,6 +128,7 @@ extern int opal_enter_rtas(struct rtas_args *args,
 #define OPAL_XSCOM_WRITE			66
 #define OPAL_LPC_READ				67
 #define OPAL_LPC_WRITE				68
+#define OPAL_RETURN_CPU				69
 
 #ifndef __ASSEMBLY__
 
@@ -646,6 +647,7 @@ int64_t opal_set_system_attention_led(uint8_t led_action);
 int64_t opal_pci_next_error(uint64_t phb_id, uint64_t *first_frozen_pe,
 			    uint16_t *pci_error_type, uint16_t *severity);
 int64_t opal_pci_poll(uint64_t phb_id);
+int64_t opal_return_cpu(void);
 
 int64_t opal_xscom_read(uint32_t gcid, uint32_t pcb_addr, uint64_t *val);
 int64_t opal_xscom_write(uint32_t gcid, uint32_t pcb_addr, uint64_t val);

arch/powerpc/platforms/powernv/opal-wrappers.S

@@ -115,3 +115,4 @@ OPAL_CALL(opal_xscom_read,			OPAL_XSCOM_READ);
 OPAL_CALL(opal_xscom_write,			OPAL_XSCOM_WRITE);
 OPAL_CALL(opal_lpc_read,			OPAL_LPC_READ);
 OPAL_CALL(opal_lpc_write,			OPAL_LPC_WRITE);
+OPAL_CALL(opal_return_cpu,			OPAL_RETURN_CPU);

arch/powerpc/platforms/powernv/setup.c

@@ -31,6 +31,7 @@
 #include <asm/xics.h>
 #include <asm/rtas.h>
 #include <asm/opal.h>
+#include <asm/kexec.h>
 
 #include "powernv.h"
 
@@ -153,6 +154,16 @@ static void pnv_shutdown(void)
 static void pnv_kexec_cpu_down(int crash_shutdown, int secondary)
 {
 	xics_kexec_teardown_cpu(secondary);
+
+	/* Return secondary CPUs to firmware on OPAL v3 */
+	if (firmware_has_feature(FW_FEATURE_OPALv3) && secondary) {
+		mb();
+		get_paca()->kexec_state = KEXEC_STATE_REAL_MODE;
+		mb();
+
+		/* Return the CPU to OPAL */
+		opal_return_cpu();
+	}
 }
 #endif /* CONFIG_KEXEC */