powerpc/eeh: Block PCI config access upon frozen PE

The problem was found when I tried to inject PCI config error by
PHB3 PAPR error injection registers into Broadcom Austin 4-ports
NIC adapter. The frozen PE was reported successfully and EEH core
started to recover it. However, I run into fenced PHB when dumping
PCI config space as EEH logs. I was told that PCI config requests
should not be progagated to the adapter until PE reset is done
successfully. Otherise, we would run out of PHB internal credits
and trigger PCT (PCIE Completion Timeout), which leads to the
fenced PHB.

The patch introduces another PE flag EEH_PE_CFG_RESTRICTED, which
is set during PE initialization time if the PE includes the specific
PCI devices that need block PCI config access until PE reset is done.
When the PE becomes frozen for the first time, EEH_PE_CFG_BLOCKED is
set if the PE has flag EEH_PE_CFG_RESTRICTED. Then the PCI config
access to the PE will be dropped by platform PCI accessors until
PE reset is done successfully. The mechanism is shared by PowerNV
platform owned PE or userland owned ones. It's not used on pSeries
platform yet.
Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

arch/powerpc/include/asm/eeh.h

@@ -74,6 +74,7 @@ struct device_node;
 #define EEH_PE_CFG_BLOCKED	(1 << 2)	/* Block config access	*/
 
 #define EEH_PE_KEEP		(1 << 8)	/* Keep PE on hotplug	*/
+#define EEH_PE_CFG_RESTRICTED	(1 << 9)	/* Block config on error */
 
 struct eeh_pe {
 	int type;			/* PE type: PHB/Bus/Device	*/

arch/powerpc/kernel/eeh_pe.c

@@ -534,6 +534,10 @@ static void *__eeh_pe_state_mark(void *data, void *flag)
 			pdev->error_state = pci_channel_io_frozen;
 	}
 
+	/* Block PCI config access if required */
+	if (pe->state & EEH_PE_CFG_RESTRICTED)
+		pe->state |= EEH_PE_CFG_BLOCKED;
+
 	return NULL;
 }
 
@@ -611,6 +615,10 @@ static void *__eeh_pe_state_clear(void *data, void *flag)
 		pdev->error_state = pci_channel_io_normal;
 	}
 
+	/* Unblock PCI config access if required */
+	if (pe->state & EEH_PE_CFG_RESTRICTED)
+		pe->state &= ~EEH_PE_CFG_BLOCKED;
+
 	return NULL;
 }
 

arch/powerpc/platforms/powernv/eeh-powernv.c

@@ -168,6 +168,25 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag)
 		return ret;
 	}
 
+	/*
+	 * If the PE contains any one of following adapters, the
+	 * PCI config space can't be accessed when dumping EEH log.
+	 * Otherwise, we will run into fenced PHB caused by shortage
+	 * of outbound credits in the adapter. The PCI config access
+	 * should be blocked until PE reset. MMIO access is dropped
+	 * by hardware certainly. In order to drop PCI config requests,
+	 * one more flag (EEH_PE_CFG_RESTRICTED) is introduced, which
+	 * will be checked in the backend for PE state retrival. If
+	 * the PE becomes frozen for the first time and the flag has
+	 * been set for the PE, we will set EEH_PE_CFG_BLOCKED for
+	 * that PE to block its config space.
+	 *
+	 * Broadcom Austin 4-ports NICs (14e4:1657)
+	 */
+	if (dev->vendor == PCI_VENDOR_ID_BROADCOM &&
+	    dev->device == 0x1657)
+		edev->pe->state |= EEH_PE_CFG_RESTRICTED;
+
 	/*
 	 * Cache the PE primary bus, which can't be fetched when
 	 * full hotplug is in progress. In that case, all child