eth: fbnic: Add register init to set PCIe/Ethernet device config

As a part of enabling the device the first step is to configure the AXI and
Ethernet interfaces to allow for basic traffic. This consists of
configuring several registers related to the PCIe and Ethernet FIFOs as
well as configuring the handlers for moving traffic between entities.
Signed-off-by: Alexander Duyck <alexanderduyck@fb.com>
Link: https://patch.msgid.link/172079936376.1778861.15942501417449077552.stgit@ahduyck-xeon-server.home.arpaSigned-off-by: Jakub Kicinski <kuba@kernel.org>

drivers/net/ethernet/meta/fbnic/Makefile

@@ -9,4 +9,5 @@ obj-$(CONFIG_FBNIC) += fbnic.o
 
 fbnic-y := fbnic_devlink.o \
 	   fbnic_irq.o \
+	   fbnic_mac.o \
 	   fbnic_pci.o

drivers/net/ethernet/meta/fbnic/fbnic.h

@@ -4,16 +4,22 @@
 #ifndef _FBNIC_H_
 #define _FBNIC_H_
 
+#include <linux/io.h>
+
 #include "fbnic_csr.h"
+#include "fbnic_mac.h"
 
 struct fbnic_dev {
 	struct device *dev;
 
 	u32 __iomem *uc_addr0;
 	u32 __iomem *uc_addr4;
+	const struct fbnic_mac *mac;
 	unsigned short num_irqs;
 
 	u64 dsn;
+	u32 mps;
+	u32 readrq;
 };
 
 /* Reserve entry 0 in the MSI-X "others" array until we have filled all
@@ -25,6 +31,41 @@ enum {
 	FBNIC_NON_NAPI_VECTORS
 };
 
+static inline bool fbnic_present(struct fbnic_dev *fbd)
+{
+	return !!READ_ONCE(fbd->uc_addr0);
+}
+
+static inline void fbnic_wr32(struct fbnic_dev *fbd, u32 reg, u32 val)
+{
+	u32 __iomem *csr = READ_ONCE(fbd->uc_addr0);
+
+	if (csr)
+		writel(val, csr + reg);
+}
+
+u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg);
+
+static inline void fbnic_wrfl(struct fbnic_dev *fbd)
+{
+	fbnic_rd32(fbd, FBNIC_MASTER_SPARE_0);
+}
+
+static inline void
+fbnic_rmw32(struct fbnic_dev *fbd, u32 reg, u32 mask, u32 val)
+{
+	u32 v;
+
+	v = fbnic_rd32(fbd, reg);
+	v &= ~mask;
+	v |= val;
+	fbnic_wr32(fbd, reg, v);
+}
+
+#define wr32(_f, _r, _v)	fbnic_wr32(_f, _r, _v)
+#define rd32(_f, _r)		fbnic_rd32(_f, _r)
+#define wrfl(_f)		fbnic_wrfl(_f)
+
 extern char fbnic_driver_name[];
 
 void fbnic_devlink_free(struct fbnic_dev *fbd);

drivers/net/ethernet/meta/fbnic/fbnic_devlink.c

@@ -65,6 +65,8 @@ struct fbnic_dev *fbnic_devlink_alloc(struct pci_dev *pdev)
 	fbd->uc_addr4 = iomap_table[4];
 
 	fbd->dsn = pci_get_dsn(pdev);
+	fbd->mps = pcie_get_mps(pdev);
+	fbd->readrq = pcie_get_readrq(pdev);
 
 	return fbd;
 }

drivers/net/ethernet/meta/fbnic/fbnic_mac.h

+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#ifndef _FBNIC_MAC_H_
+#define _FBNIC_MAC_H_
+
+#include <linux/types.h>
+
+struct fbnic_dev;
+
+#define FBNIC_MAX_JUMBO_FRAME_SIZE	9742
+
+/* This structure defines the interface hooks for the MAC. The MAC hooks
+ * will be configured as a const struct provided with a set of function
+ * pointers.
+ *
+ * void (*init_regs)(struct fbnic_dev *fbd);
+ *	Initialize MAC registers to enable Tx/Rx paths and FIFOs.
+ */
+struct fbnic_mac {
+	void (*init_regs)(struct fbnic_dev *fbd);
+};
+
+int fbnic_mac_init(struct fbnic_dev *fbd);
+#endif /* _FBNIC_MAC_H_ */

drivers/net/ethernet/meta/fbnic/fbnic_pci.c

@@ -29,6 +29,35 @@ static const struct pci_device_id fbnic_pci_tbl[] = {
 };
 MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl);
 
+u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg)
+{
+	u32 __iomem *csr = READ_ONCE(fbd->uc_addr0);
+	u32 value;
+
+	if (!csr)
+		return ~0U;
+
+	value = readl(csr + reg);
+
+	/* If any bits are 0 value should be valid */
+	if (~value)
+		return value;
+
+	/* All 1's may be valid if ZEROs register still works */
+	if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0))
+		return value;
+
+	/* Hardware is giving us all 1's reads, assume it is gone */
+	WRITE_ONCE(fbd->uc_addr0, NULL);
+	WRITE_ONCE(fbd->uc_addr4, NULL);
+
+	dev_err(fbd->dev,
+		"Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
+		reg, reg << 2);
+
+	return ~0U;
+}
+
 /**
  * fbnic_probe - Device Initialization Routine
  * @pdev: PCI device information struct
@@ -86,6 +115,12 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (err)
 		goto free_fbd;
 
+	err = fbnic_mac_init(fbd);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err);
+		goto free_irqs;
+	}
+
 	fbnic_devlink_register(fbd);
 
 	if (!fbd->dsn) {
@@ -101,6 +136,8 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	  * firmware updates for fixes.
 	  */
 	return 0;
+free_irqs:
+	fbnic_free_irqs(fbd);
 free_fbd:
 	pci_disable_device(pdev);
 	fbnic_devlink_free(fbd);
@@ -157,6 +194,8 @@ static int __fbnic_pm_resume(struct device *dev)
 	if (err)
 		goto err_invalidate_uc_addr;
 
+	fbd->mac->init_regs(fbd);
+
 	return 0;
 err_invalidate_uc_addr:
 	WRITE_ONCE(fbd->uc_addr0, NULL);