netdevsim: create devlink instance per netdevsim instance

Currently there is one devlink instance created per network namespace.
That is quite odd considering the fact that devlink instance should
represent an ASIC. The following patches are going to move the devlink
instance even more down to a bus device, but until then, have one
devlink instance per netdevsim instance. Struct nsim_devlink is
introduced to hold fib setting.

The changes in the fib code are only related to holding the
configuration per devlink instance instead of network namespace.
Signed-off-by: Jiri Pirko <jiri@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

drivers/net/netdevsim/devlink.c

@@ -15,59 +15,59 @@
  */
 
 #include <linux/device.h>
+#include <linux/rtnetlink.h>
 #include <net/devlink.h>
-#include <net/netns/generic.h>
 
 #include "netdevsim.h"
 
-static unsigned int nsim_devlink_id;
-
-/* place holder until devlink and namespaces is sorted out */
-static struct net *nsim_devlink_net(struct devlink *devlink)
-{
-	return &init_net;
-}
+struct nsim_devlink {
+	struct nsim_fib_data *fib_data;
+};
 
 /* IPv4
  */
 static u64 nsim_ipv4_fib_resource_occ_get(void *priv)
 {
-	struct net *net = priv;
+	struct nsim_devlink *nsim_devlink = priv;
 
-	return nsim_fib_get_val(net, NSIM_RESOURCE_IPV4_FIB, false);
+	return nsim_fib_get_val(nsim_devlink->fib_data,
+				NSIM_RESOURCE_IPV4_FIB, false);
 }
 
 static u64 nsim_ipv4_fib_rules_res_occ_get(void *priv)
 {
-	struct net *net = priv;
+	struct nsim_devlink *nsim_devlink = priv;
 
-	return nsim_fib_get_val(net, NSIM_RESOURCE_IPV4_FIB_RULES, false);
+	return nsim_fib_get_val(nsim_devlink->fib_data,
+				NSIM_RESOURCE_IPV4_FIB_RULES, false);
 }
 
 /* IPv6
  */
 static u64 nsim_ipv6_fib_resource_occ_get(void *priv)
 {
-	struct net *net = priv;
+	struct nsim_devlink *nsim_devlink = priv;
 
-	return nsim_fib_get_val(net, NSIM_RESOURCE_IPV6_FIB, false);
+	return nsim_fib_get_val(nsim_devlink->fib_data,
+				NSIM_RESOURCE_IPV6_FIB, false);
 }
 
 static u64 nsim_ipv6_fib_rules_res_occ_get(void *priv)
 {
-	struct net *net = priv;
+	struct nsim_devlink *nsim_devlink = priv;
 
-	return nsim_fib_get_val(net, NSIM_RESOURCE_IPV6_FIB_RULES, false);
+	return nsim_fib_get_val(nsim_devlink->fib_data,
+				NSIM_RESOURCE_IPV6_FIB_RULES, false);
 }
 
 static int devlink_resources_register(struct devlink *devlink)
 {
+	struct nsim_devlink *nsim_devlink = devlink_priv(devlink);
 	struct devlink_resource_size_params params = {
 		.size_max = (u64)-1,
 		.size_granularity = 1,
 		.unit = DEVLINK_RESOURCE_UNIT_ENTRY
 	};
-	struct net *net = nsim_devlink_net(devlink);
 	int err;
 	u64 n;
 
@@ -81,7 +81,8 @@ static int devlink_resources_register(struct devlink *devlink)
 		goto out;
 	}
 
-	n = nsim_fib_get_val(net, NSIM_RESOURCE_IPV4_FIB, true);
+	n = nsim_fib_get_val(nsim_devlink->fib_data,
+			     NSIM_RESOURCE_IPV4_FIB, true);
 	err = devlink_resource_register(devlink, "fib", n,
 					NSIM_RESOURCE_IPV4_FIB,
 					NSIM_RESOURCE_IPV4, &params);
@@ -90,7 +91,8 @@ static int devlink_resources_register(struct devlink *devlink)
 		return err;
 	}
 
-	n = nsim_fib_get_val(net, NSIM_RESOURCE_IPV4_FIB_RULES, true);
+	n = nsim_fib_get_val(nsim_devlink->fib_data,
+			     NSIM_RESOURCE_IPV4_FIB_RULES, true);
 	err = devlink_resource_register(devlink, "fib-rules", n,
 					NSIM_RESOURCE_IPV4_FIB_RULES,
 					NSIM_RESOURCE_IPV4, &params);
@@ -109,7 +111,8 @@ static int devlink_resources_register(struct devlink *devlink)
 		goto out;
 	}
 
-	n = nsim_fib_get_val(net, NSIM_RESOURCE_IPV6_FIB, true);
+	n = nsim_fib_get_val(nsim_devlink->fib_data,
+			     NSIM_RESOURCE_IPV6_FIB, true);
 	err = devlink_resource_register(devlink, "fib", n,
 					NSIM_RESOURCE_IPV6_FIB,
 					NSIM_RESOURCE_IPV6, &params);
@@ -118,7 +121,8 @@ static int devlink_resources_register(struct devlink *devlink)
 		return err;
 	}
 
-	n = nsim_fib_get_val(net, NSIM_RESOURCE_IPV6_FIB_RULES, true);
+	n = nsim_fib_get_val(nsim_devlink->fib_data,
+			     NSIM_RESOURCE_IPV6_FIB_RULES, true);
 	err = devlink_resource_register(devlink, "fib-rules", n,
 					NSIM_RESOURCE_IPV6_FIB_RULES,
 					NSIM_RESOURCE_IPV6, &params);
@@ -130,19 +134,19 @@ static int devlink_resources_register(struct devlink *devlink)
 	devlink_resource_occ_get_register(devlink,
 					  NSIM_RESOURCE_IPV4_FIB,
 					  nsim_ipv4_fib_resource_occ_get,
-					  net);
+					  nsim_devlink);
 	devlink_resource_occ_get_register(devlink,
 					  NSIM_RESOURCE_IPV4_FIB_RULES,
 					  nsim_ipv4_fib_rules_res_occ_get,
-					  net);
+					  nsim_devlink);
 	devlink_resource_occ_get_register(devlink,
 					  NSIM_RESOURCE_IPV6_FIB,
 					  nsim_ipv6_fib_resource_occ_get,
-					  net);
+					  nsim_devlink);
 	devlink_resource_occ_get_register(devlink,
 					  NSIM_RESOURCE_IPV6_FIB_RULES,
 					  nsim_ipv6_fib_rules_res_occ_get,
-					  net);
+					  nsim_devlink);
 out:
 	return err;
 }
@@ -150,11 +154,11 @@ static int devlink_resources_register(struct devlink *devlink)
 static int nsim_devlink_reload(struct devlink *devlink,
 			       struct netlink_ext_ack *extack)
 {
+	struct nsim_devlink *nsim_devlink = devlink_priv(devlink);
 	enum nsim_resource_id res_ids[] = {
 		NSIM_RESOURCE_IPV4_FIB, NSIM_RESOURCE_IPV4_FIB_RULES,
 		NSIM_RESOURCE_IPV6_FIB, NSIM_RESOURCE_IPV6_FIB_RULES
 	};
-	struct net *net = nsim_devlink_net(devlink);
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(res_ids); ++i) {
@@ -163,7 +167,8 @@ static int nsim_devlink_reload(struct devlink *devlink,
 
 		err = devlink_resource_size_get(devlink, res_ids[i], &val);
 		if (!err) {
-			err = nsim_fib_set_max(net, res_ids[i], val, extack);
+			err = nsim_fib_set_max(nsim_devlink->fib_data,
+					       res_ids[i], val, extack);
 			if (err)
 				return err;
 		}
@@ -172,124 +177,68 @@ static int nsim_devlink_reload(struct devlink *devlink,
 	return 0;
 }
 
-static void nsim_devlink_net_reset(struct net *net)
-{
-	enum nsim_resource_id res_ids[] = {
-		NSIM_RESOURCE_IPV4_FIB, NSIM_RESOURCE_IPV4_FIB_RULES,
-		NSIM_RESOURCE_IPV6_FIB, NSIM_RESOURCE_IPV6_FIB_RULES
-	};
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(res_ids); ++i) {
-		if (nsim_fib_set_max(net, res_ids[i], (u64)-1, NULL)) {
-			pr_err("Failed to reset limit for resource %u\n",
-			       res_ids[i]);
-		}
-	}
-}
-
 static const struct devlink_ops nsim_devlink_ops = {
 	.reload = nsim_devlink_reload,
 };
 
-/* once devlink / namespace issues are sorted out
- * this needs to be net in which a devlink instance
- * is to be created. e.g., dev_net(ns->netdev)
- */
-static struct net *nsim_to_net(struct netdevsim *ns)
+static int __nsim_devlink_init(struct netdevsim *ns)
 {
-	return &init_net;
-}
-
-void nsim_devlink_teardown(struct netdevsim *ns)
-{
-	if (ns->devlink) {
-		struct net *net = nsim_to_net(ns);
-		bool *reg_devlink = net_generic(net, nsim_devlink_id);
-
-		devlink_resources_unregister(ns->devlink, NULL);
-		devlink_unregister(ns->devlink);
-		devlink_free(ns->devlink);
-		ns->devlink = NULL;
-
-		nsim_devlink_net_reset(net);
-		*reg_devlink = true;
-	}
-}
-
-int nsim_devlink_setup(struct netdevsim *ns)
-{
-	struct net *net = nsim_to_net(ns);
-	bool *reg_devlink = net_generic(net, nsim_devlink_id);
+	struct nsim_devlink *nsim_devlink;
 	struct devlink *devlink;
 	int err;
 
-	/* only one device per namespace controls devlink */
-	if (!*reg_devlink) {
-		ns->devlink = NULL;
-		return 0;
-	}
-
-	devlink = devlink_alloc(&nsim_devlink_ops, 0);
+	devlink = devlink_alloc(&nsim_devlink_ops, sizeof(*nsim_devlink));
 	if (!devlink)
 		return -ENOMEM;
+	nsim_devlink = devlink_priv(devlink);
 
-	err = devlink_register(devlink, &ns->dev);
-	if (err)
+	nsim_devlink->fib_data = nsim_fib_create();
+	if (IS_ERR(nsim_devlink->fib_data)) {
+		err = PTR_ERR(nsim_devlink->fib_data);
 		goto err_devlink_free;
+	}
 
 	err = devlink_resources_register(devlink);
 	if (err)
-		goto err_dl_unregister;
+		goto err_fib_destroy;
 
-	ns->devlink = devlink;
+	err = devlink_register(devlink, &ns->dev);
+	if (err)
+		goto err_resources_unregister;
 
-	*reg_devlink = false;
+	ns->devlink = devlink;
 
 	return 0;
 
-err_dl_unregister:
-	devlink_unregister(devlink);
+err_resources_unregister:
+	devlink_resources_unregister(devlink, NULL);
+err_fib_destroy:
+	nsim_fib_destroy(nsim_devlink->fib_data);
 err_devlink_free:
 	devlink_free(devlink);
 
 	return err;
 }
 
-/* Initialize per network namespace state */
-static int __net_init nsim_devlink_netns_init(struct net *net)
+int nsim_devlink_init(struct netdevsim *ns)
 {
-	bool *reg_devlink = net_generic(net, nsim_devlink_id);
-
-	*reg_devlink = true;
-
-	return 0;
-}
-
-static struct pernet_operations nsim_devlink_net_ops = {
-	.init = nsim_devlink_netns_init,
-	.id   = &nsim_devlink_id,
-	.size = sizeof(bool),
-};
+	int err;
 
-void nsim_devlink_exit(void)
-{
-	unregister_pernet_subsys(&nsim_devlink_net_ops);
-	nsim_fib_exit();
+	dev_hold(ns->netdev);
+	rtnl_unlock();
+	err = __nsim_devlink_init(ns);
+	rtnl_lock();
+	dev_put(ns->netdev);
+	return err;
 }
 
-int nsim_devlink_init(void)
+void nsim_devlink_exit(struct netdevsim *ns)
 {
-	int err;
+	struct devlink *devlink = ns->devlink;
+	struct nsim_devlink *nsim_devlink = devlink_priv(devlink);
 
-	err = nsim_fib_init();
-	if (err)
-		goto err_out;
-
-	err = register_pernet_subsys(&nsim_devlink_net_ops);
-	if (err)
-		nsim_fib_exit();
-
-err_out:
-	return err;
+	devlink_unregister(devlink);
+	devlink_resources_unregister(devlink, NULL);
+	nsim_fib_destroy(nsim_devlink->fib_data);
+	devlink_free(devlink);
 }

drivers/net/netdevsim/fib.c

@@ -18,7 +18,6 @@
 #include <net/ip_fib.h>
 #include <net/ip6_fib.h>
 #include <net/fib_rules.h>
-#include <net/netns/generic.h>
 
 #include "netdevsim.h"
 
@@ -33,15 +32,14 @@ struct nsim_per_fib_data {
 };
 
 struct nsim_fib_data {
+	struct notifier_block fib_nb;
 	struct nsim_per_fib_data ipv4;
 	struct nsim_per_fib_data ipv6;
 };
 
-static unsigned int nsim_fib_net_id;
-
-u64 nsim_fib_get_val(struct net *net, enum nsim_resource_id res_id, bool max)
+u64 nsim_fib_get_val(struct nsim_fib_data *fib_data,
+		     enum nsim_resource_id res_id, bool max)
 {
-	struct nsim_fib_data *fib_data = net_generic(net, nsim_fib_net_id);
 	struct nsim_fib_entry *entry;
 
 	switch (res_id) {
@@ -64,10 +62,10 @@ u64 nsim_fib_get_val(struct net *net, enum nsim_resource_id res_id, bool max)
 	return max ? entry->max : entry->num;
 }
 
-int nsim_fib_set_max(struct net *net, enum nsim_resource_id res_id, u64 val,
+int nsim_fib_set_max(struct nsim_fib_data *fib_data,
+		     enum nsim_resource_id res_id, u64 val,
 		     struct netlink_ext_ack *extack)
 {
-	struct nsim_fib_data *fib_data = net_generic(net, nsim_fib_net_id);
 	struct nsim_fib_entry *entry;
 	int err = 0;
 
@@ -120,9 +118,9 @@ static int nsim_fib_rule_account(struct nsim_fib_entry *entry, bool add,
 	return err;
 }
 
-static int nsim_fib_rule_event(struct fib_notifier_info *info, bool add)
+static int nsim_fib_rule_event(struct nsim_fib_data *data,
+			       struct fib_notifier_info *info, bool add)
 {
-	struct nsim_fib_data *data = net_generic(info->net, nsim_fib_net_id);
 	struct netlink_ext_ack *extack = info->extack;
 	int err = 0;
 
@@ -157,9 +155,9 @@ static int nsim_fib_account(struct nsim_fib_entry *entry, bool add,
 	return err;
 }
 
-static int nsim_fib_event(struct fib_notifier_info *info, bool add)
+static int nsim_fib_event(struct nsim_fib_data *data,
+			  struct fib_notifier_info *info, bool add)
 {
-	struct nsim_fib_data *data = net_generic(info->net, nsim_fib_net_id);
 	struct netlink_ext_ack *extack = info->extack;
 	int err = 0;
 
@@ -178,18 +176,22 @@ static int nsim_fib_event(struct fib_notifier_info *info, bool add)
 static int nsim_fib_event_nb(struct notifier_block *nb, unsigned long event,
 			     void *ptr)
 {
+	struct nsim_fib_data *data = container_of(nb, struct nsim_fib_data,
+						  fib_nb);
 	struct fib_notifier_info *info = ptr;
 	int err = 0;
 
 	switch (event) {
 	case FIB_EVENT_RULE_ADD: /* fall through */
 	case FIB_EVENT_RULE_DEL:
-		err = nsim_fib_rule_event(info, event == FIB_EVENT_RULE_ADD);
+		err = nsim_fib_rule_event(data, info,
+					  event == FIB_EVENT_RULE_ADD);
 		break;
 
 	case FIB_EVENT_ENTRY_ADD:  /* fall through */
 	case FIB_EVENT_ENTRY_DEL:
-		err = nsim_fib_event(info, event == FIB_EVENT_ENTRY_ADD);
+		err = nsim_fib_event(data, info,
+				     event == FIB_EVENT_ENTRY_ADD);
 		break;
 	}
 
@@ -199,30 +201,23 @@ static int nsim_fib_event_nb(struct notifier_block *nb, unsigned long event,
 /* inconsistent dump, trying again */
 static void nsim_fib_dump_inconsistent(struct notifier_block *nb)
 {
-	struct nsim_fib_data *data;
-	struct net *net;
-
-	rcu_read_lock();
-	for_each_net_rcu(net) {
-		data = net_generic(net, nsim_fib_net_id);
+	struct nsim_fib_data *data = container_of(nb, struct nsim_fib_data,
+						  fib_nb);
 
 	data->ipv4.fib.num = 0ULL;
 	data->ipv4.rules.num = 0ULL;
-
 	data->ipv6.fib.num = 0ULL;
 	data->ipv6.rules.num = 0ULL;
-	}
-	rcu_read_unlock();
 }
 
-static struct notifier_block nsim_fib_nb = {
-	.notifier_call = nsim_fib_event_nb,
-};
-
-/* Initialize per network namespace state */
-static int __net_init nsim_fib_netns_init(struct net *net)
+struct nsim_fib_data *nsim_fib_create(void)
 {
-	struct nsim_fib_data *data = net_generic(net, nsim_fib_net_id);
+	struct nsim_fib_data *data;
+	int err;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return ERR_PTR(-ENOMEM);
 
 	data->ipv4.fib.max = (u64)-1;
 	data->ipv4.rules.max = (u64)-1;
@@ -230,37 +225,22 @@ static int __net_init nsim_fib_netns_init(struct net *net)
 	data->ipv6.fib.max = (u64)-1;
 	data->ipv6.rules.max = (u64)-1;
 
-	return 0;
-}
-
-static struct pernet_operations nsim_fib_net_ops = {
-	.init = nsim_fib_netns_init,
-	.id   = &nsim_fib_net_id,
-	.size = sizeof(struct nsim_fib_data),
-};
-
-void nsim_fib_exit(void)
-{
-	unregister_pernet_subsys(&nsim_fib_net_ops);
-	unregister_fib_notifier(&nsim_fib_nb);
-}
-
-int nsim_fib_init(void)
-{
-	int err;
-
-	err = register_pernet_subsys(&nsim_fib_net_ops);
-	if (err < 0) {
-		pr_err("Failed to register pernet subsystem\n");
-		goto err_out;
-	}
-
-	err = register_fib_notifier(&nsim_fib_nb, nsim_fib_dump_inconsistent);
-	if (err < 0) {
+	data->fib_nb.notifier_call = nsim_fib_event_nb;
+	err = register_fib_notifier(&data->fib_nb, nsim_fib_dump_inconsistent);
+	if (err) {
 		pr_err("Failed to register fib notifier\n");
 		goto err_out;
 	}
 
+	return data;
+
 err_out:
-	return err;
+	kfree(data);
+	return ERR_PTR(err);
+}
+
+void nsim_fib_destroy(struct nsim_fib_data *data)
+{
+	unregister_fib_notifier(&data->fib_nb);
+	kfree(data);
 }

drivers/net/netdevsim/netdev.c

@@ -161,7 +161,6 @@ static int nsim_init(struct net_device *dev)
 	char sdev_link_name[32];
 	int err;
 
-	ns->netdev = dev;
 	ns->ddir = debugfs_create_dir(netdev_name(dev), nsim_ddir);
 	if (IS_ERR_OR_NULL(ns->ddir))
 		return -ENOMEM;
@@ -174,16 +173,10 @@ static int nsim_init(struct net_device *dev)
 	if (err)
 		goto err_debugfs_destroy;
 
-	err = nsim_devlink_setup(ns);
-	if (err)
-		goto err_bpf_uninit;
-
 	nsim_ipsec_init(ns);
 
 	return 0;
 
-err_bpf_uninit:
-	nsim_bpf_uninit(ns);
 err_debugfs_destroy:
 	debugfs_remove_recursive(ns->ddir);
 	return err;
@@ -194,7 +187,6 @@ static void nsim_uninit(struct net_device *dev)
 	struct netdevsim *ns = netdev_priv(dev);
 
 	nsim_ipsec_teardown(ns);
-	nsim_devlink_teardown(ns);
 	debugfs_remove_recursive(ns->ddir);
 	nsim_bpf_uninit(ns);
 }
@@ -203,6 +195,7 @@ static void nsim_free(struct net_device *dev)
 {
 	struct netdevsim *ns = netdev_priv(dev);
 
+	nsim_devlink_exit(ns);
 	device_unregister(&ns->dev);
 	/* netdev and vf state will be freed out of device_release() */
 	nsim_sdev_put(ns->sdev);
@@ -511,12 +504,19 @@ static int nsim_newlink(struct net *src_net, struct net_device *dev,
 		goto err_sdev_put;
 
 	SET_NETDEV_DEV(dev, &ns->dev);
+	ns->netdev = dev;
 
-	err = register_netdevice(dev);
+	err = nsim_devlink_init(ns);
 	if (err)
 		goto err_unreg_dev;
+
+	err = register_netdevice(dev);
+	if (err)
+		goto err_devlink_exit;
 	return 0;
 
+err_devlink_exit:
+	nsim_devlink_exit(ns);
 err_unreg_dev:
 	device_unregister(&ns->dev);
 err_sdev_put:
@@ -548,18 +548,12 @@ static int __init nsim_module_init(void)
 	if (err)
 		goto err_sdev_exit;
 
-	err = nsim_devlink_init();
-	if (err)
-		goto err_unreg_bus;
-
 	err = rtnl_link_register(&nsim_link_ops);
 	if (err)
-		goto err_dl_fini;
+		goto err_unreg_bus;
 
 	return 0;
 
-err_dl_fini:
-	nsim_devlink_exit();
 err_unreg_bus:
 	bus_unregister(&nsim_bus);
 err_sdev_exit:
@@ -572,7 +566,6 @@ static int __init nsim_module_init(void)
 static void __exit nsim_module_exit(void)
 {
 	rtnl_link_unregister(&nsim_link_ops);
-	nsim_devlink_exit();
 	bus_unregister(&nsim_bus);
 	nsim_sdev_exit();
 	debugfs_remove_recursive(nsim_ddir);

drivers/net/netdevsim/netdevsim.h

@@ -30,6 +30,7 @@ struct bpf_prog;
 struct bpf_offload_dev;
 struct dentry;
 struct nsim_vf_config;
+struct nsim_fib_data;
 
 struct netdevsim_shared_dev {
 	unsigned int refcnt;
@@ -153,16 +154,15 @@ enum nsim_resource_id {
 	NSIM_RESOURCE_IPV6_FIB_RULES,
 };
 
-int nsim_devlink_setup(struct netdevsim *ns);
-void nsim_devlink_teardown(struct netdevsim *ns);
+int nsim_devlink_init(struct netdevsim *ns);
+void nsim_devlink_exit(struct netdevsim *ns);
 
-int nsim_devlink_init(void);
-void nsim_devlink_exit(void);
-
-int nsim_fib_init(void);
-void nsim_fib_exit(void);
-u64 nsim_fib_get_val(struct net *net, enum nsim_resource_id res_id, bool max);
-int nsim_fib_set_max(struct net *net, enum nsim_resource_id res_id, u64 val,
+struct nsim_fib_data *nsim_fib_create(void);
+void nsim_fib_destroy(struct nsim_fib_data *fib_data);
+u64 nsim_fib_get_val(struct nsim_fib_data *fib_data,
+		     enum nsim_resource_id res_id, bool max);
+int nsim_fib_set_max(struct nsim_fib_data *fib_data,
+		     enum nsim_resource_id res_id, u64 val,
 		     struct netlink_ext_ack *extack);
 
 #if IS_ENABLED(CONFIG_XFRM_OFFLOAD)