Commit fadd59fc authored by Aviv Heller's avatar Aviv Heller Committed by Saeed Mahameed

net/mlx5: Introduce inter-device communication mechanism

This introduces devcom, a generic mechanism for performing operations
on both physical functions of the same Connect-X card.

The first user of this API is merged eswitch, which will be introduced
in subsequent patches.
Signed-off-by: default avatarAviv Heller <avivh@mellanox.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
parent 64e4cf0d
...@@ -15,7 +15,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ ...@@ -15,7 +15,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
health.o mcg.o cq.o alloc.o qp.o port.o mr.o pd.o \ health.o mcg.o cq.o alloc.o qp.o port.o mr.o pd.o \
mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o \ mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o \
fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \ fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \
diag/fs_tracepoint.o diag/fw_tracer.o lib/devcom.o diag/fs_tracepoint.o diag/fw_tracer.o
# #
# Netdev basic # Netdev basic
......
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2018 Mellanox Technologies */
#include <linux/mlx5/vport.h>
#include "lib/devcom.h"
static LIST_HEAD(devcom_list);
#define devcom_for_each_component(priv, comp, iter) \
for (iter = 0; \
comp = &(priv)->components[iter], iter < MLX5_DEVCOM_NUM_COMPONENTS; \
iter++)
struct mlx5_devcom_component {
struct {
void *data;
} device[MLX5_MAX_PORTS];
mlx5_devcom_event_handler_t handler;
struct rw_semaphore sem;
bool paired;
};
struct mlx5_devcom_list {
struct list_head list;
struct mlx5_devcom_component components[MLX5_DEVCOM_NUM_COMPONENTS];
struct mlx5_core_dev *devs[MLX5_MAX_PORTS];
};
struct mlx5_devcom {
struct mlx5_devcom_list *priv;
int idx;
};
static struct mlx5_devcom_list *mlx5_devcom_list_alloc(void)
{
struct mlx5_devcom_component *comp;
struct mlx5_devcom_list *priv;
int i;
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return NULL;
devcom_for_each_component(priv, comp, i)
init_rwsem(&comp->sem);
return priv;
}
static struct mlx5_devcom *mlx5_devcom_alloc(struct mlx5_devcom_list *priv,
u8 idx)
{
struct mlx5_devcom *devcom;
devcom = kzalloc(sizeof(*devcom), GFP_KERNEL);
if (!devcom)
return NULL;
devcom->priv = priv;
devcom->idx = idx;
return devcom;
}
/* Must be called with intf_mutex held */
struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev)
{
struct mlx5_devcom_list *priv = NULL, *iter;
struct mlx5_devcom *devcom = NULL;
bool new_priv = false;
u64 sguid0, sguid1;
int idx, i;
if (!mlx5_core_is_pf(dev))
return NULL;
sguid0 = mlx5_query_nic_system_image_guid(dev);
list_for_each_entry(iter, &devcom_list, list) {
struct mlx5_core_dev *tmp_dev = NULL;
idx = -1;
for (i = 0; i < MLX5_MAX_PORTS; i++) {
if (iter->devs[i])
tmp_dev = iter->devs[i];
else
idx = i;
}
if (idx == -1)
continue;
sguid1 = mlx5_query_nic_system_image_guid(tmp_dev);
if (sguid0 != sguid1)
continue;
priv = iter;
break;
}
if (!priv) {
priv = mlx5_devcom_list_alloc();
if (!priv)
return ERR_PTR(-ENOMEM);
idx = 0;
new_priv = true;
}
priv->devs[idx] = dev;
devcom = mlx5_devcom_alloc(priv, idx);
if (!devcom) {
kfree(priv);
return ERR_PTR(-ENOMEM);
}
if (new_priv)
list_add(&priv->list, &devcom_list);
return devcom;
}
/* Must be called with intf_mutex held */
void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom)
{
struct mlx5_devcom_list *priv;
int i;
if (IS_ERR_OR_NULL(devcom))
return;
priv = devcom->priv;
priv->devs[devcom->idx] = NULL;
kfree(devcom);
for (i = 0; i < MLX5_MAX_PORTS; i++)
if (priv->devs[i])
break;
if (i != MLX5_MAX_PORTS)
return;
list_del(&priv->list);
kfree(priv);
}
void mlx5_devcom_register_component(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id,
mlx5_devcom_event_handler_t handler,
void *data)
{
struct mlx5_devcom_component *comp;
if (IS_ERR_OR_NULL(devcom))
return;
WARN_ON(!data);
comp = &devcom->priv->components[id];
down_write(&comp->sem);
comp->handler = handler;
comp->device[devcom->idx].data = data;
up_write(&comp->sem);
}
void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id)
{
struct mlx5_devcom_component *comp;
if (IS_ERR_OR_NULL(devcom))
return;
comp = &devcom->priv->components[id];
down_write(&comp->sem);
comp->device[devcom->idx].data = NULL;
up_write(&comp->sem);
}
int mlx5_devcom_send_event(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id,
int event,
void *event_data)
{
struct mlx5_devcom_component *comp;
int err = -ENODEV, i;
if (IS_ERR_OR_NULL(devcom))
return err;
comp = &devcom->priv->components[id];
down_write(&comp->sem);
for (i = 0; i < MLX5_MAX_PORTS; i++)
if (i != devcom->idx && comp->device[i].data) {
err = comp->handler(event, comp->device[i].data,
event_data);
break;
}
up_write(&comp->sem);
return err;
}
void mlx5_devcom_set_paired(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id,
bool paired)
{
struct mlx5_devcom_component *comp;
comp = &devcom->priv->components[id];
WARN_ON(!rwsem_is_locked(&comp->sem));
comp->paired = paired;
}
bool mlx5_devcom_is_paired(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id)
{
if (IS_ERR_OR_NULL(devcom))
return false;
return devcom->priv->components[id].paired;
}
void *mlx5_devcom_get_peer_data(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id)
{
struct mlx5_devcom_component *comp;
int i;
if (IS_ERR_OR_NULL(devcom))
return NULL;
comp = &devcom->priv->components[id];
down_read(&comp->sem);
if (!comp->paired) {
up_read(&comp->sem);
return NULL;
}
for (i = 0; i < MLX5_MAX_PORTS; i++)
if (i != devcom->idx)
break;
return comp->device[i].data;
}
void mlx5_devcom_release_peer_data(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id)
{
struct mlx5_devcom_component *comp = &devcom->priv->components[id];
up_read(&comp->sem);
}
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2018 Mellanox Technologies */
#ifndef __LIB_MLX5_DEVCOM_H__
#define __LIB_MLX5_DEVCOM_H__
#include <linux/mlx5/driver.h>
enum mlx5_devcom_components {
MLX5_DEVCOM_NUM_COMPONENTS,
};
typedef int (*mlx5_devcom_event_handler_t)(int event,
void *my_data,
void *event_data);
struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev);
void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom);
void mlx5_devcom_register_component(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id,
mlx5_devcom_event_handler_t handler,
void *data);
void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id);
int mlx5_devcom_send_event(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id,
int event,
void *event_data);
void mlx5_devcom_set_paired(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id,
bool paired);
bool mlx5_devcom_is_paired(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id);
void *mlx5_devcom_get_peer_data(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id);
void mlx5_devcom_release_peer_data(struct mlx5_devcom *devcom,
enum mlx5_devcom_components id);
#endif
...@@ -63,6 +63,7 @@ ...@@ -63,6 +63,7 @@
#include "accel/tls.h" #include "accel/tls.h"
#include "lib/clock.h" #include "lib/clock.h"
#include "lib/vxlan.h" #include "lib/vxlan.h"
#include "lib/devcom.h"
#include "diag/fw_tracer.h" #include "diag/fw_tracer.h"
MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
...@@ -722,16 +723,21 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) ...@@ -722,16 +723,21 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
struct pci_dev *pdev = dev->pdev; struct pci_dev *pdev = dev->pdev;
int err; int err;
priv->devcom = mlx5_devcom_register_device(dev);
if (IS_ERR(priv->devcom))
dev_err(&pdev->dev, "failed to register with devcom (0x%p)\n",
priv->devcom);
err = mlx5_query_board_id(dev); err = mlx5_query_board_id(dev);
if (err) { if (err) {
dev_err(&pdev->dev, "query board id failed\n"); dev_err(&pdev->dev, "query board id failed\n");
goto out; goto err_devcom;
} }
err = mlx5_eq_table_init(dev); err = mlx5_eq_table_init(dev);
if (err) { if (err) {
dev_err(&pdev->dev, "failed to initialize eq\n"); dev_err(&pdev->dev, "failed to initialize eq\n");
goto out; goto err_devcom;
} }
err = mlx5_events_init(dev); err = mlx5_events_init(dev);
...@@ -807,8 +813,9 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) ...@@ -807,8 +813,9 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
mlx5_events_cleanup(dev); mlx5_events_cleanup(dev);
err_eq_cleanup: err_eq_cleanup:
mlx5_eq_table_cleanup(dev); mlx5_eq_table_cleanup(dev);
err_devcom:
mlx5_devcom_unregister_device(dev->priv.devcom);
out:
return err; return err;
} }
...@@ -828,6 +835,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) ...@@ -828,6 +835,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
mlx5_cq_debugfs_cleanup(dev); mlx5_cq_debugfs_cleanup(dev);
mlx5_events_cleanup(dev); mlx5_events_cleanup(dev);
mlx5_eq_table_cleanup(dev); mlx5_eq_table_cleanup(dev);
mlx5_devcom_unregister_device(dev->priv.devcom);
} }
static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
......
...@@ -486,6 +486,7 @@ struct mlx5_events; ...@@ -486,6 +486,7 @@ struct mlx5_events;
struct mlx5_mpfs; struct mlx5_mpfs;
struct mlx5_eswitch; struct mlx5_eswitch;
struct mlx5_lag; struct mlx5_lag;
struct mlx5_devcom;
struct mlx5_eq_table; struct mlx5_eq_table;
struct mlx5_rate_limit { struct mlx5_rate_limit {
...@@ -560,6 +561,7 @@ struct mlx5_priv { ...@@ -560,6 +561,7 @@ struct mlx5_priv {
struct mlx5_eswitch *eswitch; struct mlx5_eswitch *eswitch;
struct mlx5_core_sriov sriov; struct mlx5_core_sriov sriov;
struct mlx5_lag *lag; struct mlx5_lag *lag;
struct mlx5_devcom *devcom;
unsigned long pci_dev_data; unsigned long pci_dev_data;
struct mlx5_fc_stats fc_stats; struct mlx5_fc_stats fc_stats;
struct mlx5_rl_table rl_table; struct mlx5_rl_table rl_table;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment