net/mlx5: E-Switch, Reg/unreg function changed event at correct stage

When driver is doing eswitch mode change, it's critical to keep number
of enabled VFs unchanged. However, it can be changed on the fly once
function changed event is registered.

To remove this uncertainty, function changed event should not be
registered before all setups, and first be unregistered before all
cleanups. Wrap this functionality together with vport event handler.

Fixes: 61fc880839e6 ("net/mlx5: E-Switch, Handle representors creation in handler context")
Signed-off-by: Bodong Wang <bodong@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>

drivers/net/ethernet/mellanox/mlx5/core/eswitch.c

@@ -1725,6 +1725,28 @@ int mlx5_esw_query_functions(struct mlx5_core_dev *dev, u32 *out, int outlen)
 	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 }
 
+static void mlx5_eswitch_event_handlers_register(struct mlx5_eswitch *esw)
+{
+	if (esw->mode == MLX5_ESWITCH_LEGACY) {
+		MLX5_NB_INIT(&esw->nb, eswitch_vport_event, NIC_VPORT_CHANGE);
+		mlx5_eq_notifier_register(esw->dev, &esw->nb);
+	} else if (mlx5_eswitch_is_funcs_handler(esw->dev)) {
+		MLX5_NB_INIT(&esw->esw_funcs.nb, mlx5_esw_funcs_changed_handler,
+			     ESW_FUNCTIONS_CHANGED);
+		mlx5_eq_notifier_register(esw->dev, &esw->esw_funcs.nb);
+	}
+}
+
+static void mlx5_eswitch_event_handlers_unregister(struct mlx5_eswitch *esw)
+{
+	if (esw->mode == MLX5_ESWITCH_LEGACY)
+		mlx5_eq_notifier_unregister(esw->dev, &esw->nb);
+	else if (mlx5_eswitch_is_funcs_handler(esw->dev))
+		mlx5_eq_notifier_unregister(esw->dev, &esw->esw_funcs.nb);
+
+	flush_workqueue(esw->work_queue);
+}
+
 /* Public E-Switch API */
 #define ESW_ALLOWED(esw) ((esw) && MLX5_ESWITCH_MANAGER((esw)->dev))
 
@@ -1787,10 +1809,7 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int mode)
 	mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
 		esw_enable_vport(esw, vport, enabled_events);
 
-	if (mode == MLX5_ESWITCH_LEGACY) {
-		MLX5_NB_INIT(&esw->nb, eswitch_vport_event, NIC_VPORT_CHANGE);
-		mlx5_eq_notifier_register(esw->dev, &esw->nb);
-	}
+	mlx5_eswitch_event_handlers_register(esw);
 
 	esw_info(esw->dev, "Enable: mode(%s), nvfs(%d), active vports(%d)\n",
 		 mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",
@@ -1824,9 +1843,7 @@ void mlx5_eswitch_disable(struct mlx5_eswitch *esw)
 		 esw->esw_funcs.num_vfs, esw->enabled_vports);
 
 	mc_promisc = &esw->mc_promisc;
-
-	if (esw->mode == MLX5_ESWITCH_LEGACY)
-		mlx5_eq_notifier_unregister(esw->dev, &esw->nb);
+	mlx5_eswitch_event_handlers_unregister(esw);
 
 	mlx5_esw_for_all_vports(esw, i, vport)
 		esw_disable_vport(esw, vport);

drivers/net/ethernet/mellanox/mlx5/core/eswitch.h

@@ -524,6 +524,7 @@ mlx5_eswitch_get_vport(struct mlx5_eswitch *esw, u16 vport_num);
 bool mlx5_eswitch_is_vf_vport(const struct mlx5_eswitch *esw, u16 vport_num);
 
 void mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, const int num_vfs);
+int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data);
 
 #else  /* CONFIG_MLX5_ESWITCH */
 /* eswitch API stubs */

drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c

@@ -2083,9 +2083,7 @@ static void esw_functions_changed_event_handler(struct work_struct *work)
 	kfree(host_work);
 }
 
-
-static int
-esw_functions_changed_event(struct notifier_block *nb, unsigned long type, void *data)
+int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
 {
 	struct mlx5_esw_functions *esw_funcs;
 	struct mlx5_host_work *host_work;
@@ -2106,24 +2104,6 @@ esw_functions_changed_event(struct notifier_block *nb, unsigned long type, void
 	return NOTIFY_OK;
 }
 
-static void esw_functions_changed_event_init(struct mlx5_eswitch *esw)
-{
-	if (mlx5_eswitch_is_funcs_handler(esw->dev)) {
-		MLX5_NB_INIT(&esw->esw_funcs.nb, esw_functions_changed_event,
-			     ESW_FUNCTIONS_CHANGED);
-		mlx5_eq_notifier_register(esw->dev, &esw->esw_funcs.nb);
-	}
-}
-
-static void esw_functions_changed_event_cleanup(struct mlx5_eswitch *esw)
-{
-	if (!mlx5_eswitch_is_funcs_handler(esw->dev))
-		return;
-
-	mlx5_eq_notifier_unregister(esw->dev, &esw->esw_funcs.nb);
-	flush_workqueue(esw->work_queue);
-}
-
 int esw_offloads_init(struct mlx5_eswitch *esw)
 {
 	int err;
@@ -2144,8 +2124,6 @@ int esw_offloads_init(struct mlx5_eswitch *esw)
 
 	esw_offloads_devcom_init(esw);
 
-	esw_functions_changed_event_init(esw);
-
 	mlx5_rdma_enable_roce(esw->dev);
 
 	return 0;
@@ -2179,7 +2157,6 @@ static int esw_offloads_stop(struct mlx5_eswitch *esw,
 
 void esw_offloads_cleanup(struct mlx5_eswitch *esw)
 {
-	esw_functions_changed_event_cleanup(esw);
 	mlx5_rdma_disable_roce(esw->dev);
 	esw_offloads_devcom_cleanup(esw);
 	esw_offloads_unload_all_reps(esw);