nfp: abm: spawn port netdevs

To configure buffering points we need full set of netdevs:

                              ASIC

 user netdev  -- | -- PCIe port   MAC port -- | --

Configuring egrees qdiscs on user netdev configures standard
Linux TC software qdiscs, configuring PCIe port qdiscs will
provide a way of setting ASIC queuing parameters for PCIe block.
MAC port netdev egress qdiscs correspond to ASIC MAC Traffic
Manager block.
Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Reviewed-by: Dirk van der Merwe <dirk.vandermerwe@netronome.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

drivers/net/ethernet/netronome/nfp/abm/main.c

@@ -32,7 +32,12 @@
  * SOFTWARE.
  */
 
+#include <linux/bitfield.h>
 #include <linux/etherdevice.h>
+#include <linux/lockdep.h>
+#include <linux/netdevice.h>
+#include <linux/rcupdate.h>
+#include <linux/slab.h>
 
 #include "../nfpcore/nfp.h"
 #include "../nfpcore/nfp_cpp.h"
@@ -40,9 +45,195 @@
 #include "../nfp_app.h"
 #include "../nfp_main.h"
 #include "../nfp_net.h"
+#include "../nfp_net_repr.h"
 #include "../nfp_port.h"
 #include "main.h"
 
+static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id)
+{
+	return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) |
+	       FIELD_PREP(NFP_ABM_PORTID_ID, id);
+}
+
+static struct net_device *nfp_abm_repr_get(struct nfp_app *app, u32 port_id)
+{
+	enum nfp_repr_type rtype;
+	struct nfp_reprs *reprs;
+	u8 port;
+
+	rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id);
+	port = FIELD_GET(NFP_ABM_PORTID_ID, port_id);
+
+	reprs = rcu_dereference(app->reprs[rtype]);
+	if (!reprs)
+		return NULL;
+
+	if (port >= reprs->num_reprs)
+		return NULL;
+
+	return rcu_dereference(reprs->reprs[port]);
+}
+
+static int
+nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink,
+		   enum nfp_port_type ptype)
+{
+	struct net_device *netdev;
+	enum nfp_repr_type rtype;
+	struct nfp_reprs *reprs;
+	struct nfp_repr *repr;
+	struct nfp_port *port;
+	int err;
+
+	if (ptype == NFP_PORT_PHYS_PORT)
+		rtype = NFP_REPR_TYPE_PHYS_PORT;
+	else
+		rtype = NFP_REPR_TYPE_PF;
+
+	netdev = nfp_repr_alloc(app);
+	if (!netdev)
+		return -ENOMEM;
+	repr = netdev_priv(netdev);
+	repr->app_priv = alink;
+
+	port = nfp_port_alloc(app, ptype, netdev);
+	if (IS_ERR(port)) {
+		err = PTR_ERR(port);
+		goto err_free_repr;
+	}
+
+	if (ptype == NFP_PORT_PHYS_PORT) {
+		err = nfp_port_init_phy_port(app->pf, app, port, alink->id);
+		if (err)
+			goto err_free_port;
+	} else {
+		port->pf_id = alink->abm->pf_id;
+		port->vnic = alink->vnic->dp.ctrl_bar;
+	}
+
+	SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev);
+	eth_hw_addr_random(netdev);
+
+	err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id),
+			    port, alink->vnic->dp.netdev);
+	if (err)
+		goto err_free_port;
+
+	reprs = nfp_reprs_get_locked(app, rtype);
+	WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr");
+	rcu_assign_pointer(reprs->reprs[alink->id], netdev);
+
+	nfp_info(app->cpp, "%s Port %d Representor(%s) created\n",
+		 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys",
+		 alink->id, netdev->name);
+
+	return 0;
+
+err_free_port:
+	nfp_port_free(port);
+err_free_repr:
+	nfp_repr_free(netdev);
+	return err;
+}
+
+static void
+nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink,
+		  enum nfp_repr_type rtype)
+{
+	struct net_device *netdev;
+	struct nfp_reprs *reprs;
+
+	reprs = nfp_reprs_get_locked(app, rtype);
+	netdev = nfp_repr_get_locked(app, reprs, alink->id);
+	if (!netdev)
+		return;
+	rcu_assign_pointer(reprs->reprs[alink->id], NULL);
+	synchronize_rcu();
+	/* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */
+	nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev));
+}
+
+static void
+nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink)
+{
+	nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF);
+	nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT);
+}
+
+static void nfp_abm_kill_reprs_all(struct nfp_abm *abm)
+{
+	struct nfp_pf *pf = abm->app->pf;
+	struct nfp_net *nn;
+
+	list_for_each_entry(nn, &pf->vnics, vnic_list)
+		nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv);
+}
+
+static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app)
+{
+	struct nfp_abm *abm = app->priv;
+
+	return abm->eswitch_mode;
+}
+
+static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm)
+{
+	nfp_abm_kill_reprs_all(abm);
+
+	abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
+	return 0;
+}
+
+static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm)
+{
+	if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY)
+		WARN_ON(nfp_abm_eswitch_set_legacy(abm));
+}
+
+static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm)
+{
+	struct nfp_app *app = abm->app;
+	struct nfp_pf *pf = app->pf;
+	struct nfp_net *nn;
+	int err;
+
+	list_for_each_entry(nn, &pf->vnics, vnic_list) {
+		struct nfp_abm_link *alink = nn->app_priv;
+
+		err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT);
+		if (err)
+			goto err_kill_all_reprs;
+
+		err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT);
+		if (err)
+			goto err_kill_all_reprs;
+	}
+
+	abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
+	return 0;
+
+err_kill_all_reprs:
+	nfp_abm_kill_reprs_all(abm);
+	return err;
+}
+
+static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode)
+{
+	struct nfp_abm *abm = app->priv;
+
+	if (abm->eswitch_mode == mode)
+		return 0;
+
+	switch (mode) {
+	case DEVLINK_ESWITCH_MODE_LEGACY:
+		return nfp_abm_eswitch_set_legacy(abm);
+	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
+		return nfp_abm_eswitch_set_switchdev(abm);
+	default:
+		return -EINVAL;
+	}
+}
+
 static void
 nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn,
 		     unsigned int id)
@@ -87,7 +278,6 @@ nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
 {
 	struct nfp_abm *abm = app->priv;
 	struct nfp_abm_link *alink;
-	int err;
 
 	alink = kzalloc(sizeof(*alink), GFP_KERNEL);
 	if (!alink)
@@ -97,41 +287,26 @@ nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
 	alink->vnic = nn;
 	alink->id = id;
 
-	nn->port = nfp_port_alloc(app, NFP_PORT_PHYS_PORT, nn->dp.netdev);
-	if (IS_ERR(nn->port)) {
-		err = PTR_ERR(nn->port);
-		goto err_free_alink;
-	}
-
-	err = nfp_app_nic_vnic_init_phy_port(app->pf, app, nn, id);
-	if (err < 0)
-		goto err_free_port;
-	if (nn->port->type == NFP_PORT_INVALID)
-		/* core will kill this vNIC */
-		return 0;
+	netif_keep_dst(nn->dp.netdev);
 
 	nfp_abm_vnic_set_mac(app->pf, abm, nn, id);
 	nfp_abm_ctrl_read_params(alink);
 
 	return 0;
-
-err_free_port:
-	nfp_port_free(nn->port);
-err_free_alink:
-	kfree(alink);
-	return err;
 }
 
 static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn)
 {
 	struct nfp_abm_link *alink = nn->app_priv;
 
+	nfp_abm_kill_reprs(alink->abm, alink);
 	kfree(alink);
 }
 
 static int nfp_abm_init(struct nfp_app *app)
 {
 	struct nfp_pf *pf = app->pf;
+	struct nfp_reprs *reprs;
 	struct nfp_abm *abm;
 	int err;
 
@@ -159,8 +334,21 @@ static int nfp_abm_init(struct nfp_app *app)
 	if (err)
 		goto err_free_abm;
 
+	err = -ENOMEM;
+	reprs = nfp_reprs_alloc(pf->max_data_vnics);
+	if (!reprs)
+		goto err_free_abm;
+	RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs);
+
+	reprs = nfp_reprs_alloc(pf->max_data_vnics);
+	if (!reprs)
+		goto err_free_phys;
+	RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs);
+
 	return 0;
 
+err_free_phys:
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
 err_free_abm:
 	kfree(abm);
 	app->priv = NULL;
@@ -171,6 +359,9 @@ static void nfp_abm_clean(struct nfp_app *app)
 {
 	struct nfp_abm *abm = app->priv;
 
+	nfp_abm_eswitch_clean_up(abm);
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
 	kfree(abm);
 	app->priv = NULL;
 }
@@ -184,4 +375,9 @@ const struct nfp_app_type app_abm = {
 
 	.vnic_alloc	= nfp_abm_vnic_alloc,
 	.vnic_free	= nfp_abm_vnic_free,
+
+	.eswitch_mode_get	= nfp_abm_eswitch_mode_get,
+	.eswitch_mode_set	= nfp_abm_eswitch_mode_set,
+
+	.repr_get	= nfp_abm_repr_get,
 };

drivers/net/ethernet/netronome/nfp/abm/main.h

@@ -35,17 +35,25 @@
 #ifndef __NFP_ABM_H__
 #define __NFP_ABM_H__ 1
 
+#include <net/devlink.h>
+
 struct nfp_app;
 struct nfp_net;
 
+#define NFP_ABM_PORTID_TYPE	GENMASK(23, 16)
+#define NFP_ABM_PORTID_ID	GENMASK(7, 0)
+
 /**
  * struct nfp_abm - ABM NIC app structure
  * @app:	back pointer to nfp_app
  * @pf_id:	ID of our PF link
+ * @eswitch_mode:	devlink eswitch mode, advanced functions only visible
+ *			in switchdev mode
  */
 struct nfp_abm {
 	struct nfp_app *app;
 	unsigned int pf_id;
+	enum devlink_eswitch_mode eswitch_mode;
 };
 
 /**

drivers/net/ethernet/netronome/nfp/nfp_net_repr.c

@@ -385,7 +385,7 @@ struct net_device *nfp_repr_alloc(struct nfp_app *app)
 	return NULL;
 }
 
-static void nfp_repr_clean_and_free(struct nfp_repr *repr)
+void nfp_repr_clean_and_free(struct nfp_repr *repr)
 {
 	nfp_info(repr->app->cpp, "Destroying Representor(%s)\n",
 		 repr->netdev->name);

drivers/net/ethernet/netronome/nfp/nfp_net_repr.h

@@ -127,6 +127,7 @@ int nfp_repr_init(struct nfp_app *app, struct net_device *netdev,
 		  struct net_device *pf_netdev);
 void nfp_repr_free(struct net_device *netdev);
 struct net_device *nfp_repr_alloc(struct nfp_app *app);
+void nfp_repr_clean_and_free(struct nfp_repr *repr);
 void nfp_reprs_clean_and_free(struct nfp_app *app, struct nfp_reprs *reprs);
 void nfp_reprs_clean_and_free_by_type(struct nfp_app *app,
 				      enum nfp_repr_type type);