Commit 1a676d2b authored by David S. Miller's avatar David S. Miller

Merge branch 'sfc-next'

Shradha Shah says:

====================
sfc: Enabling EF10 Vf's, set up vswitching and bind the SFC driver to the VF's

This set of patches makes way for the implementation of EF10
SR-IOV driver starting with some cleanup code.
NIC specific SR-IOV functions are moved to their own header
and netdev_ops are made generic instead of being NIC specific

Next in line comes the patch to enable VF's using sriov_configure.
VEB vswitching hierarchy is set up next followed by patches to
prepare sfc driver to bind to enabled VF's

This is followed by patch to support use of shared RSS contexts
which makes VF's use shared RSS contexts in all cases.

Patch series ends with a patch to bind the sfc driver to the
enabled VF's which creates network interfaces corresponding to
the VF's.

Coming up soon are the patches to set_vf_mac, set_vf_config,
set_vf_vlan, vf_spoofcheck, etc.

These patches have been tested with and without CONFIG_SFC_SRIOV.
In the case of CONFIG_SFC_SRIOV=y enabling of VF's using
sriov_configure is also tested. The enabled VF's bind to the
installed sfc driver succesfully to create network interfaces.
In the case of CONFIG_SFC_SRIOV=n enabling of VF's using
sriov_configure returns the correct error message:
"Function not implemented".
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents f926204b 6f7f8aa6
......@@ -3,6 +3,6 @@ sfc-y += efx.o nic.o farch.o falcon.o siena.o ef10.o tx.o \
tenxpress.o txc43128_phy.o falcon_boards.o \
mcdi.o mcdi_port.o mcdi_mon.o ptp.o
sfc-$(CONFIG_SFC_MTD) += mtd.o
sfc-$(CONFIG_SFC_SRIOV) += siena_sriov.o
sfc-$(CONFIG_SFC_SRIOV) += sriov.o siena_sriov.o ef10_sriov.o
obj-$(CONFIG_SFC) += sfc.o
This diff is collapsed.
This diff is collapsed.
/****************************************************************************
* Driver for Solarflare network controllers and boards
* Copyright 2015 Solarflare Communications Inc.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation, incorporated herein by reference.
*/
#ifndef EF10_SRIOV_H
#define EF10_SRIOV_H
#include "net_driver.h"
/**
* struct ef10_vf - PF's store of VF data
* @vport_id: vport ID for the VF
* @vport_assigned: record whether the vport is currently assigned to the VF
* @mac: MAC address for the VF, zero when address is removed from the vport
*/
struct ef10_vf {
unsigned int vport_id;
unsigned int vport_assigned;
u8 mac[ETH_ALEN];
};
static inline bool efx_ef10_sriov_wanted(struct efx_nic *efx)
{
return false;
}
int efx_ef10_sriov_configure(struct efx_nic *efx, int num_vfs);
int efx_ef10_sriov_init(struct efx_nic *efx);
static inline void efx_ef10_sriov_mac_address_changed(struct efx_nic *efx) {}
static inline void efx_ef10_sriov_reset(struct efx_nic *efx) {}
void efx_ef10_sriov_fini(struct efx_nic *efx);
static inline void efx_ef10_sriov_flr(struct efx_nic *efx, unsigned vf_i) {}
static inline int efx_ef10_sriov_set_vf_mac(struct efx_nic *efx, int vf,
u8 *mac)
{
return -EOPNOTSUPP;
}
static inline int efx_ef10_sriov_set_vf_vlan(struct efx_nic *efx, int vf,
u16 vlan, u8 qos)
{
return -EOPNOTSUPP;
}
static inline int efx_ef10_sriov_set_vf_spoofchk(struct efx_nic *efx, int vf,
bool spoofchk)
{
return -EOPNOTSUPP;
}
static inline int efx_ef10_sriov_get_vf_config(struct efx_nic *efx, int vf,
struct ifla_vf_info *ivf)
{
return -EOPNOTSUPP;
}
int efx_ef10_vswitching_probe_pf(struct efx_nic *efx);
int efx_ef10_vswitching_probe_vf(struct efx_nic *efx);
int efx_ef10_vswitching_restore_pf(struct efx_nic *efx);
int efx_ef10_vswitching_restore_vf(struct efx_nic *efx);
void efx_ef10_vswitching_remove_pf(struct efx_nic *efx);
void efx_ef10_vswitching_remove_vf(struct efx_nic *efx);
#endif /* EF10_SRIOV_H */
This diff is collapsed.
......@@ -15,7 +15,9 @@
#include "filter.h"
/* All controllers use BAR 0 for I/O space and BAR 2(&3) for memory */
/* All VFs use BAR 0/1 for memory */
#define EFX_MEM_BAR 2
#define EFX_MEM_VF_BAR 0
/* TX */
int efx_probe_tx_queue(struct efx_tx_queue *tx_queue);
......@@ -32,6 +34,7 @@ unsigned int efx_tx_max_skb_descs(struct efx_nic *efx);
extern unsigned int efx_piobuf_size;
/* RX */
void efx_set_default_rx_indir_table(struct efx_nic *efx);
void efx_rx_config_page_split(struct efx_nic *efx);
int efx_probe_rx_queue(struct efx_rx_queue *rx_queue);
void efx_remove_rx_queue(struct efx_rx_queue *rx_queue);
......@@ -220,6 +223,13 @@ static inline void efx_mtd_rename(struct efx_nic *efx) {}
static inline void efx_mtd_remove(struct efx_nic *efx) {}
#endif
#ifdef CONFIG_SFC_SRIOV
static inline unsigned int efx_vf_size(struct efx_nic *efx)
{
return 1 << efx->vi_scale;
}
#endif
static inline void efx_schedule_channel(struct efx_channel *channel)
{
netif_vdbg(channel->efx, intr, channel->efx->net_dev,
......
......@@ -1109,9 +1109,8 @@ static int efx_ethtool_set_rxfh(struct net_device *net_dev, const u32 *indir,
return -EOPNOTSUPP;
if (!indir)
return 0;
memcpy(efx->rx_indir_table, indir, sizeof(efx->rx_indir_table));
efx->type->rx_push_rss_config(efx);
return 0;
return efx->type->rx_push_rss_config(efx, true, indir);
}
static int efx_ethtool_get_ts_info(struct net_device *net_dev,
......
......@@ -477,16 +477,29 @@ static irqreturn_t falcon_legacy_interrupt_a1(int irq, void *dev_id)
*
**************************************************************************
*/
static int dummy_rx_push_rss_config(struct efx_nic *efx, bool user,
const u32 *rx_indir_table)
{
(void) efx;
(void) user;
(void) rx_indir_table;
return -ENOSYS;
}
static void falcon_b0_rx_push_rss_config(struct efx_nic *efx)
static int falcon_b0_rx_push_rss_config(struct efx_nic *efx, bool user,
const u32 *rx_indir_table)
{
efx_oword_t temp;
(void) user;
/* Set hash key for IPv4 */
memcpy(&temp, efx->rx_hash_key, sizeof(temp));
efx_writeo(efx, &temp, FR_BZ_RX_RSS_TKEY);
memcpy(efx->rx_indir_table, rx_indir_table,
sizeof(efx->rx_indir_table));
efx_farch_rx_push_indir_table(efx);
return 0;
}
/**************************************************************************
......@@ -2507,7 +2520,7 @@ static int falcon_init_nic(struct efx_nic *efx)
falcon_init_rx_cfg(efx);
if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
falcon_b0_rx_push_rss_config(efx);
falcon_b0_rx_push_rss_config(efx, false, efx->rx_indir_table);
/* Set destination of both TX and RX Flush events */
EFX_POPULATE_OWORD_1(temp, FRF_BZ_FLS_EVQ_ID, 0);
......@@ -2687,6 +2700,8 @@ static int falcon_set_wol(struct efx_nic *efx, u32 type)
*/
const struct efx_nic_type falcon_a1_nic_type = {
.is_vf = false,
.mem_bar = EFX_MEM_BAR,
.mem_map_size = falcon_a1_mem_map_size,
.probe = falcon_probe_nic,
.remove = falcon_remove_nic,
......@@ -2729,7 +2744,7 @@ const struct efx_nic_type falcon_a1_nic_type = {
.tx_init = efx_farch_tx_init,
.tx_remove = efx_farch_tx_remove,
.tx_write = efx_farch_tx_write,
.rx_push_rss_config = efx_port_dummy_op_void,
.rx_push_rss_config = dummy_rx_push_rss_config,
.rx_probe = efx_farch_rx_probe,
.rx_init = efx_farch_rx_init,
.rx_remove = efx_farch_rx_remove,
......@@ -2766,11 +2781,6 @@ const struct efx_nic_type falcon_a1_nic_type = {
.mtd_write = falcon_mtd_write,
.mtd_sync = falcon_mtd_sync,
#endif
.sriov_init = efx_falcon_sriov_init,
.sriov_fini = efx_falcon_sriov_fini,
.sriov_mac_address_changed = efx_falcon_sriov_mac_address_changed,
.sriov_wanted = efx_falcon_sriov_wanted,
.sriov_reset = efx_falcon_sriov_reset,
.revision = EFX_REV_FALCON_A1,
.txd_ptr_tbl_base = FR_AA_TX_DESC_PTR_TBL_KER,
......@@ -2788,6 +2798,8 @@ const struct efx_nic_type falcon_a1_nic_type = {
};
const struct efx_nic_type falcon_b0_nic_type = {
.is_vf = false,
.mem_bar = EFX_MEM_BAR,
.mem_map_size = falcon_b0_mem_map_size,
.probe = falcon_probe_nic,
.remove = falcon_remove_nic,
......@@ -2867,11 +2879,6 @@ const struct efx_nic_type falcon_b0_nic_type = {
.mtd_write = falcon_mtd_write,
.mtd_sync = falcon_mtd_sync,
#endif
.sriov_init = efx_falcon_sriov_init,
.sriov_fini = efx_falcon_sriov_fini,
.sriov_mac_address_changed = efx_falcon_sriov_mac_address_changed,
.sriov_wanted = efx_falcon_sriov_wanted,
.sriov_reset = efx_falcon_sriov_reset,
.revision = EFX_REV_FALCON_B0,
.txd_ptr_tbl_base = FR_BZ_TX_DESC_PTR_TBL,
......
......@@ -20,6 +20,8 @@
#include "efx.h"
#include "nic.h"
#include "farch_regs.h"
#include "sriov.h"
#include "siena_sriov.h"
#include "io.h"
#include "workarounds.h"
......@@ -1198,13 +1200,17 @@ efx_farch_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n",
channel->channel, ev_sub_data);
efx_farch_handle_tx_flush_done(efx, event);
#ifdef CONFIG_SFC_SRIOV
efx_siena_sriov_tx_flush_done(efx, event);
#endif
break;
case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n",
channel->channel, ev_sub_data);
efx_farch_handle_rx_flush_done(efx, event);
#ifdef CONFIG_SFC_SRIOV
efx_siena_sriov_rx_flush_done(efx, event);
#endif
break;
case FSE_AZ_EVQ_INIT_DONE_EV:
netif_dbg(efx, hw, efx->net_dev,
......@@ -1242,8 +1248,11 @@ efx_farch_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
" RX Q %d is disabled.\n", ev_sub_data,
ev_sub_data);
efx_schedule_reset(efx, RESET_TYPE_DMA_ERROR);
} else
}
#ifdef CONFIG_SFC_SRIOV
else
efx_siena_sriov_desc_fetch_err(efx, ev_sub_data);
#endif
break;
case FSE_BZ_TX_DSC_ERROR_EV:
if (ev_sub_data < EFX_VI_BASE) {
......@@ -1252,8 +1261,11 @@ efx_farch_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
" TX Q %d is disabled.\n", ev_sub_data,
ev_sub_data);
efx_schedule_reset(efx, RESET_TYPE_DMA_ERROR);
} else
}
#ifdef CONFIG_SFC_SRIOV
else
efx_siena_sriov_desc_fetch_err(efx, ev_sub_data);
#endif
break;
default:
netif_vdbg(efx, hw, efx->net_dev,
......@@ -1317,9 +1329,11 @@ int efx_farch_ev_process(struct efx_channel *channel, int budget)
case FSE_AZ_EV_CODE_DRIVER_EV:
efx_farch_handle_driver_event(channel, &event);
break;
#ifdef CONFIG_SFC_SRIOV
case FSE_CZ_EV_CODE_USER_EV:
efx_siena_sriov_event(channel, &event);
break;
#endif
case FSE_CZ_EV_CODE_MCDI_EV:
efx_mcdi_process_event(channel, &event);
break;
......@@ -1685,8 +1699,10 @@ void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)
vi_count = max(efx->n_channels, efx->n_tx_channels * EFX_TXQ_TYPES);
#ifdef CONFIG_SFC_SRIOV
if (efx->type->sriov_wanted) {
if (efx->type->sriov_wanted(efx)) {
unsigned vi_dc_entries, buftbl_free, entries_per_vf, vf_limit;
unsigned vi_dc_entries, buftbl_free;
unsigned entries_per_vf, vf_limit;
nic_data->vf_buftbl_base = buftbl_min;
......@@ -1695,7 +1711,8 @@ void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)
buftbl_free = (sram_lim_qw - buftbl_min -
vi_count * vi_dc_entries);
entries_per_vf = ((vi_dc_entries + EFX_VF_BUFTBL_PER_VI) *
entries_per_vf = ((vi_dc_entries +
EFX_VF_BUFTBL_PER_VI) *
efx_vf_size(efx));
vf_limit = min(buftbl_free / entries_per_vf,
(1024U - EFX_VI_BASE) >> efx->vi_scale);
......@@ -1708,6 +1725,7 @@ void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)
}
vi_count += efx->vf_count * efx_vf_size(efx);
}
}
#endif
efx->tx_dc_base = sram_lim_qw - vi_count * TX_DC_ENTRIES;
......
......@@ -1035,7 +1035,9 @@ void efx_mcdi_process_event(struct efx_channel *channel,
/* MAC stats are gather lazily. We can ignore this. */
break;
case MCDI_EVENT_CODE_FLR:
efx_siena_sriov_flr(efx, MCDI_EVENT_FIELD(*event, FLR_VF));
if (efx->type->sriov_flr)
efx->type->sriov_flr(efx,
MCDI_EVENT_FIELD(*event, FLR_VF));
break;
case MCDI_EVENT_CODE_PTP_RX:
case MCDI_EVENT_CODE_PTP_FAULT:
......@@ -1081,9 +1083,7 @@ void efx_mcdi_process_event(struct efx_channel *channel,
void efx_mcdi_print_fwver(struct efx_nic *efx, char *buf, size_t len)
{
MCDI_DECLARE_BUF(outbuf,
max(MC_CMD_GET_VERSION_OUT_LEN,
MC_CMD_GET_CAPABILITIES_OUT_LEN));
MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_VERSION_OUT_LEN);
size_t outlength;
const __le16 *ver_words;
size_t offset;
......@@ -1108,19 +1108,11 @@ void efx_mcdi_print_fwver(struct efx_nic *efx, char *buf, size_t len)
* single version. Report which variants are running.
*/
if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) {
BUILD_BUG_ON(MC_CMD_GET_CAPABILITIES_IN_LEN != 0);
rc = efx_mcdi_rpc(efx, MC_CMD_GET_CAPABILITIES, NULL, 0,
outbuf, sizeof(outbuf), &outlength);
if (rc || outlength < MC_CMD_GET_CAPABILITIES_OUT_LEN)
offset += snprintf(
buf + offset, len - offset, " rx? tx?");
else
offset += snprintf(
buf + offset, len - offset, " rx%x tx%x",
MCDI_WORD(outbuf,
GET_CAPABILITIES_OUT_RX_DPCPU_FW_ID),
MCDI_WORD(outbuf,
GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID));
struct efx_ef10_nic_data *nic_data = efx->nic_data;
offset += snprintf(buf + offset, len - offset, " rx%x tx%x",
nic_data->rx_dpcpu_fw_id,
nic_data->tx_dpcpu_fw_id);
/* It's theoretically possible for the string to exceed 31
* characters, though in practice the first three version
......@@ -1150,10 +1142,26 @@ static int efx_mcdi_drv_attach(struct efx_nic *efx, bool driver_operating,
MCDI_SET_DWORD(inbuf, DRV_ATTACH_IN_UPDATE, 1);
MCDI_SET_DWORD(inbuf, DRV_ATTACH_IN_FIRMWARE_ID, MC_CMD_FW_LOW_LATENCY);
rc = efx_mcdi_rpc(efx, MC_CMD_DRV_ATTACH, inbuf, sizeof(inbuf),
rc = efx_mcdi_rpc_quiet(efx, MC_CMD_DRV_ATTACH, inbuf, sizeof(inbuf),
outbuf, sizeof(outbuf), &outlen);
if (rc)
/* If we're not the primary PF, trying to ATTACH with a FIRMWARE_ID
* specified will fail with EPERM, and we have to tell the MC we don't
* care what firmware we get.
*/
if (rc == -EPERM) {
netif_dbg(efx, probe, efx->net_dev,
"efx_mcdi_drv_attach with fw-variant setting failed EPERM, trying without it\n");
MCDI_SET_DWORD(inbuf, DRV_ATTACH_IN_FIRMWARE_ID,
MC_CMD_FW_DONT_CARE);
rc = efx_mcdi_rpc_quiet(efx, MC_CMD_DRV_ATTACH, inbuf,
sizeof(inbuf), outbuf, sizeof(outbuf),
&outlen);
}
if (rc) {
efx_mcdi_display_error(efx, MC_CMD_DRV_ATTACH, sizeof(inbuf),
outbuf, outlen, rc);
goto fail;
}
if (outlen < MC_CMD_DRV_ATTACH_OUT_LEN) {
rc = -EIO;
goto fail;
......@@ -1178,16 +1186,6 @@ static int efx_mcdi_drv_attach(struct efx_nic *efx, bool driver_operating,
* and are completely trusted by firmware. Abort probing
* if that's not true for this function.
*/
if (driver_operating &&
(efx->mcdi->fn_flags &
(1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL |
1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_TRUSTED)) !=
(1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL |
1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_TRUSTED)) {
netif_err(efx, probe, efx->net_dev,
"This driver version only supports one function per port\n");
return -ENODEV;
}
if (was_attached != NULL)
*was_attached = MCDI_DWORD(outbuf, DRV_ATTACH_OUT_OLD_STATE);
......@@ -1385,6 +1383,9 @@ int efx_mcdi_nvram_test_all(struct efx_nic *efx)
return rc;
}
/* Returns 1 if an assertion was read, 0 if no assertion had fired,
* negative on error.
*/
static int efx_mcdi_read_assertion(struct efx_nic *efx)
{
MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_ASSERTS_IN_LEN);
......@@ -1406,6 +1407,8 @@ static int efx_mcdi_read_assertion(struct efx_nic *efx)
rc = efx_mcdi_rpc_quiet(efx, MC_CMD_GET_ASSERTS,
inbuf, MC_CMD_GET_ASSERTS_IN_LEN,
outbuf, sizeof(outbuf), &outlen);
if (rc == -EPERM)
return 0;
} while ((rc == -EINTR || rc == -EIO) && retry-- > 0);
if (rc) {
......@@ -1443,24 +1446,31 @@ static int efx_mcdi_read_assertion(struct efx_nic *efx)
MCDI_ARRAY_DWORD(outbuf, GET_ASSERTS_OUT_GP_REGS_OFFS,
index));
return 0;
return 1;
}
static void efx_mcdi_exit_assertion(struct efx_nic *efx)
static int efx_mcdi_exit_assertion(struct efx_nic *efx)
{
MCDI_DECLARE_BUF(inbuf, MC_CMD_REBOOT_IN_LEN);
int rc;
/* If the MC is running debug firmware, it might now be
* waiting for a debugger to attach, but we just want it to
* reboot. We set a flag that makes the command a no-op if it
* has already done so. We don't know what return code to
* expect (0 or -EIO), so ignore it.
* has already done so.
* The MCDI will thus return either 0 or -EIO.
*/
BUILD_BUG_ON(MC_CMD_REBOOT_OUT_LEN != 0);
MCDI_SET_DWORD(inbuf, REBOOT_IN_FLAGS,
MC_CMD_REBOOT_FLAGS_AFTER_ASSERTION);
(void) efx_mcdi_rpc(efx, MC_CMD_REBOOT, inbuf, MC_CMD_REBOOT_IN_LEN,
rc = efx_mcdi_rpc_quiet(efx, MC_CMD_REBOOT, inbuf, MC_CMD_REBOOT_IN_LEN,
NULL, 0, NULL);
if (rc == -EIO)
rc = 0;
if (rc)
efx_mcdi_display_error(efx, MC_CMD_REBOOT, MC_CMD_REBOOT_IN_LEN,
NULL, 0, rc);
return rc;
}
int efx_mcdi_handle_assertion(struct efx_nic *efx)
......@@ -1468,12 +1478,10 @@ int efx_mcdi_handle_assertion(struct efx_nic *efx)
int rc;
rc = efx_mcdi_read_assertion(efx);
if (rc)
if (rc <= 0)
return rc;
efx_mcdi_exit_assertion(efx);
return 0;
return efx_mcdi_exit_assertion(efx);
}
void efx_mcdi_set_id_led(struct efx_nic *efx, enum efx_led_mode mode)
......@@ -1688,6 +1696,36 @@ int efx_mcdi_set_workaround(struct efx_nic *efx, u32 type, bool enabled)
NULL, 0, NULL);
}
int efx_mcdi_get_workarounds(struct efx_nic *efx, unsigned int *impl_out,
unsigned int *enabled_out)
{
MCDI_DECLARE_BUF_OUT_OR_ERR(outbuf, MC_CMD_GET_WORKAROUNDS_OUT_LEN);
size_t outlen;
int rc;
rc = efx_mcdi_rpc(efx, MC_CMD_GET_WORKAROUNDS, NULL, 0,
outbuf, sizeof(outbuf), &outlen);
if (rc)
goto fail;
if (outlen < MC_CMD_GET_WORKAROUNDS_OUT_LEN) {
rc = -EIO;
goto fail;
}
if (impl_out)
*impl_out = MCDI_DWORD(outbuf, GET_WORKAROUNDS_OUT_IMPLEMENTED);
if (enabled_out)
*enabled_out = MCDI_DWORD(outbuf, GET_WORKAROUNDS_OUT_ENABLED);
return 0;
fail:
netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
return rc;
}
#ifdef CONFIG_SFC_MTD
#define EFX_MCDI_NVRAM_LEN_MAX 128
......
......@@ -339,6 +339,8 @@ bool efx_mcdi_mac_check_fault(struct efx_nic *efx);
enum reset_type efx_mcdi_map_reset_reason(enum reset_type reason);
int efx_mcdi_reset(struct efx_nic *efx, enum reset_type method);
int efx_mcdi_set_workaround(struct efx_nic *efx, u32 type, bool enabled);
int efx_mcdi_get_workarounds(struct efx_nic *efx, unsigned int *impl_out,
unsigned int *enabled_out);
#ifdef CONFIG_SFC_MCDI_MON
int efx_mcdi_mon_probe(struct efx_nic *efx);
......
......@@ -1875,6 +1875,8 @@
#define MC_CMD_FW_FULL_FEATURED 0x0
/* enum: Prefer to use firmware with fewer features but lower latency */
#define MC_CMD_FW_LOW_LATENCY 0x1
/* enum: Only this option is allowed for non-admin functions */
#define MC_CMD_FW_DONT_CARE 0xffffffff
/* MC_CMD_DRV_ATTACH_OUT msgresponse */
#define MC_CMD_DRV_ATTACH_OUT_LEN 4
......@@ -4086,6 +4088,27 @@
#define LICENSED_APP_ID_ID_WIDTH 32
/***********************************/
/* MC_CMD_GET_WORKAROUNDS
* Read the list of all implemented and all currently enabled workarounds. The
* enums here must correspond with those in MC_CMD_WORKAROUND.
*/
#define MC_CMD_GET_WORKAROUNDS 0x59
/* MC_CMD_GET_WORKAROUNDS_OUT msgresponse */
#define MC_CMD_GET_WORKAROUNDS_OUT_LEN 8
/* Each workaround is represented by a single bit according to the enums below.
*/
#define MC_CMD_GET_WORKAROUNDS_OUT_IMPLEMENTED_OFST 0
#define MC_CMD_GET_WORKAROUNDS_OUT_ENABLED_OFST 4
/* enum: Bug 17230 work around. */
#define MC_CMD_GET_WORKAROUNDS_OUT_BUG17230 0x2
/* enum: Bug 35388 work around (unsafe EVQ writes). */
#define MC_CMD_GET_WORKAROUNDS_OUT_BUG35388 0x4
/* enum: Bug35017 workaround (A64 tables must be identity map) */
#define MC_CMD_GET_WORKAROUNDS_OUT_BUG35017 0x8
/***********************************/
/* MC_CMD_READ_REGS
* Get a dump of the MCPU registers
......
......@@ -793,7 +793,6 @@ union efx_multicast_hash {
efx_oword_t oword[EFX_MCAST_HASH_ENTRIES / sizeof(efx_oword_t) / 8];
};
struct efx_vf;
struct vfdi_status;
/**
......@@ -909,7 +908,6 @@ struct vfdi_status;
* completed (either success or failure). Not used when MCDI is used to
* flush receive queues.
* @flush_wq: wait queue used by efx_nic_flush_queues() to wait for flush completions.
* @vf: Array of &struct efx_vf objects.
* @vf_count: Number of VFs intended to be enabled.
* @vf_init_count: Number of VFs that have been fully initialised.
* @vi_scale: log2 number of vnics per VF.
......@@ -1053,7 +1051,6 @@ struct efx_nic {
wait_queue_head_t flush_wq;
#ifdef CONFIG_SFC_SRIOV
struct efx_vf *vf;
unsigned vf_count;
unsigned vf_init_count;
unsigned vi_scale;
......@@ -1092,6 +1089,7 @@ struct efx_mtd_partition {
/**
* struct efx_nic_type - Efx device type definition
* @mem_bar: Get the memory BAR
* @mem_map_size: Get memory BAR mapped size
* @probe: Probe the controller
* @remove: Free resources allocated by probe()
......@@ -1226,6 +1224,8 @@ struct efx_mtd_partition {
* @hwtstamp_filters: Mask of hardware timestamp filter types supported
*/
struct efx_nic_type {
bool is_vf;
unsigned int mem_bar;
unsigned int (*mem_map_size)(struct efx_nic *efx);
int (*probe)(struct efx_nic *efx);
void (*remove)(struct efx_nic *efx);
......@@ -1277,7 +1277,8 @@ struct efx_nic_type {
void (*tx_init)(struct efx_tx_queue *tx_queue);
void (*tx_remove)(struct efx_tx_queue *tx_queue);
void (*tx_write)(struct efx_tx_queue *tx_queue);
void (*rx_push_rss_config)(struct efx_nic *efx);
int (*rx_push_rss_config)(struct efx_nic *efx, bool user,
const u32 *rx_indir_table);
int (*rx_probe)(struct efx_rx_queue *rx_queue);
void (*rx_init)(struct efx_rx_queue *rx_queue);
void (*rx_remove)(struct efx_rx_queue *rx_queue);
......@@ -1330,11 +1331,23 @@ struct efx_nic_type {
int (*ptp_set_ts_sync_events)(struct efx_nic *efx, bool en, bool temp);
int (*ptp_set_ts_config)(struct efx_nic *efx,
struct hwtstamp_config *init);
int (*sriov_configure)(struct efx_nic *efx, int num_vfs);
int (*sriov_init)(struct efx_nic *efx);
void (*sriov_fini)(struct efx_nic *efx);
void (*sriov_mac_address_changed)(struct efx_nic *efx);
bool (*sriov_wanted)(struct efx_nic *efx);
void (*sriov_reset)(struct efx_nic *efx);
void (*sriov_flr)(struct efx_nic *efx, unsigned vf_i);
int (*sriov_set_vf_mac)(struct efx_nic *efx, int vf_i, u8 *mac);
int (*sriov_set_vf_vlan)(struct efx_nic *efx, int vf_i, u16 vlan,
u8 qos);
int (*sriov_set_vf_spoofchk)(struct efx_nic *efx, int vf_i,
bool spoofchk);
int (*sriov_get_vf_config)(struct efx_nic *efx, int vf_i,
struct ifla_vf_info *ivi);
int (*vswitching_probe)(struct efx_nic *efx);
int (*vswitching_restore)(struct efx_nic *efx);
void (*vswitching_remove)(struct efx_nic *efx);
int revision;
unsigned int txd_ptr_tbl_base;
......
......@@ -381,6 +381,7 @@ enum {
* @efx: Pointer back to main interface structure
* @wol_filter_id: Wake-on-LAN packet filter id
* @stats: Hardware statistics
* @vf: Array of &struct siena_vf objects
* @vf_buftbl_base: The zeroth buffer table index used to back VF queues.
* @vfdi_status: Common VFDI status page to be dmad to VF address space.
* @local_addr_list: List of local addresses. Protected by %local_lock.
......@@ -394,6 +395,7 @@ struct siena_nic_data {
int wol_filter_id;
u64 stats[SIENA_STAT_COUNT];
#ifdef CONFIG_SFC_SRIOV
struct siena_vf *vf;
struct efx_channel *vfdi_channel;
unsigned vf_buftbl_base;
struct efx_buffer vfdi_status;
......@@ -483,12 +485,21 @@ enum {
* @must_restore_piobufs: Flag: PIO buffers have yet to be restored after MC
* reboot
* @rx_rss_context: Firmware handle for our RSS context
* @rx_rss_context_exclusive: Whether our RSS context is exclusive or shared
* @stats: Hardware statistics
* @workaround_35388: Flag: firmware supports workaround for bug 35388
* @must_check_datapath_caps: Flag: @datapath_caps needs to be revalidated
* after MC reboot
* @datapath_caps: Capabilities of datapath firmware (FLAGS1 field of
* %MC_CMD_GET_CAPABILITIES response)
* @rx_dpcpu_fw_id: Firmware ID of the RxDPCPU
* @tx_dpcpu_fw_id: Firmware ID of the TxDPCPU
* @vport_id: The function's vport ID, only relevant for PFs
* @must_probe_vswitching: Flag: vswitching has yet to be setup after MC reboot
* @pf_index: The number for this PF, or the parent PF if this is a VF
#ifdef CONFIG_SFC_SRIOV
* @vf: Pointer to VF data structure
#endif
*/
struct efx_ef10_nic_data {
struct efx_buffer mcdi_buf;
......@@ -503,126 +514,25 @@ struct efx_ef10_nic_data {
unsigned int piobuf_handle[EF10_TX_PIOBUF_COUNT];
bool must_restore_piobufs;
u32 rx_rss_context;
bool rx_rss_context_exclusive;
u64 stats[EF10_STAT_COUNT];
bool workaround_35388;
bool must_check_datapath_caps;
u32 datapath_caps;
};
/*
* On the SFC9000 family each port is associated with 1 PCI physical
* function (PF) handled by sfc and a configurable number of virtual
* functions (VFs) that may be handled by some other driver, often in
* a VM guest. The queue pointer registers are mapped in both PF and
* VF BARs such that an 8K region provides access to a single RX, TX
* and event queue (collectively a Virtual Interface, VI or VNIC).
*
* The PF has access to all 1024 VIs while VFs are mapped to VIs
* according to VI_BASE and VI_SCALE: VF i has access to VIs numbered
* in range [VI_BASE + i << VI_SCALE, VI_BASE + i + 1 << VI_SCALE).
* The number of VIs and the VI_SCALE value are configurable but must
* be established at boot time by firmware.
*/
/* Maximum VI_SCALE parameter supported by Siena */
#define EFX_VI_SCALE_MAX 6
/* Base VI to use for SR-IOV. Must be aligned to (1 << EFX_VI_SCALE_MAX),
* so this is the smallest allowed value. */
#define EFX_VI_BASE 128U
/* Maximum number of VFs allowed */
#define EFX_VF_COUNT_MAX 127
/* Limit EVQs on VFs to be only 8k to reduce buffer table reservation */
#define EFX_MAX_VF_EVQ_SIZE 8192UL
/* The number of buffer table entries reserved for each VI on a VF */
#define EFX_VF_BUFTBL_PER_VI \
((EFX_MAX_VF_EVQ_SIZE + 2 * EFX_MAX_DMAQ_SIZE) * \
sizeof(efx_qword_t) / EFX_BUF_SIZE)
unsigned int rx_dpcpu_fw_id;
unsigned int tx_dpcpu_fw_id;
unsigned int vport_id;
bool must_probe_vswitching;
unsigned int pf_index;
#ifdef CONFIG_SFC_SRIOV
/* SIENA */
static inline bool efx_siena_sriov_wanted(struct efx_nic *efx)
{
return efx->vf_count != 0;
}
static inline bool efx_siena_sriov_enabled(struct efx_nic *efx)
{
return efx->vf_init_count != 0;
}
static inline unsigned int efx_vf_size(struct efx_nic *efx)
{
return 1 << efx->vi_scale;
}
struct ef10_vf *vf;
#endif
u8 vport_mac[ETH_ALEN];
};
int efx_init_sriov(void);
void efx_siena_sriov_probe(struct efx_nic *efx);
int efx_siena_sriov_init(struct efx_nic *efx);
void efx_siena_sriov_mac_address_changed(struct efx_nic *efx);
void efx_siena_sriov_tx_flush_done(struct efx_nic *efx, efx_qword_t *event);
void efx_siena_sriov_rx_flush_done(struct efx_nic *efx, efx_qword_t *event);
void efx_siena_sriov_event(struct efx_channel *channel, efx_qword_t *event);
void efx_siena_sriov_desc_fetch_err(struct efx_nic *efx, unsigned dmaq);
void efx_siena_sriov_flr(struct efx_nic *efx, unsigned flr);
void efx_siena_sriov_reset(struct efx_nic *efx);
void efx_siena_sriov_fini(struct efx_nic *efx);
void efx_fini_sriov(void);
/* EF10 */
static inline bool efx_ef10_sriov_wanted(struct efx_nic *efx) { return false; }
static inline int efx_ef10_sriov_init(struct efx_nic *efx) { return -EOPNOTSUPP; }
static inline void efx_ef10_sriov_mac_address_changed(struct efx_nic *efx) {}
static inline void efx_ef10_sriov_reset(struct efx_nic *efx) {}
static inline void efx_ef10_sriov_fini(struct efx_nic *efx) {}
#else
/* SIENA */
static inline bool efx_siena_sriov_wanted(struct efx_nic *efx) { return false; }
static inline bool efx_siena_sriov_enabled(struct efx_nic *efx) { return false; }
static inline unsigned int efx_vf_size(struct efx_nic *efx) { return 0; }
static inline int efx_init_sriov(void) { return 0; }
static inline void efx_siena_sriov_probe(struct efx_nic *efx) {}
static inline int efx_siena_sriov_init(struct efx_nic *efx) { return -EOPNOTSUPP; }
static inline void efx_siena_sriov_mac_address_changed(struct efx_nic *efx) {}
static inline void efx_siena_sriov_tx_flush_done(struct efx_nic *efx,
efx_qword_t *event) {}
static inline void efx_siena_sriov_rx_flush_done(struct efx_nic *efx,
efx_qword_t *event) {}
static inline void efx_siena_sriov_event(struct efx_channel *channel,
efx_qword_t *event) {}
static inline void efx_siena_sriov_desc_fetch_err(struct efx_nic *efx,
unsigned dmaq) {}
static inline void efx_siena_sriov_flr(struct efx_nic *efx, unsigned flr) {}
static inline void efx_siena_sriov_reset(struct efx_nic *efx) {}
static inline void efx_siena_sriov_fini(struct efx_nic *efx) {}
static inline void efx_fini_sriov(void) {}
/* EF10 */
static inline bool efx_ef10_sriov_wanted(struct efx_nic *efx) { return false; }
static inline int efx_ef10_sriov_init(struct efx_nic *efx) { return -EOPNOTSUPP; }
static inline void efx_ef10_sriov_mac_address_changed(struct efx_nic *efx) {}
static inline void efx_ef10_sriov_reset(struct efx_nic *efx) {}
static inline void efx_ef10_sriov_fini(struct efx_nic *efx) {}
#endif
/* FALCON */
static inline bool efx_falcon_sriov_wanted(struct efx_nic *efx) { return false; }
static inline int efx_falcon_sriov_init(struct efx_nic *efx) { return -EOPNOTSUPP; }
static inline void efx_falcon_sriov_mac_address_changed(struct efx_nic *efx) {}
static inline void efx_falcon_sriov_reset(struct efx_nic *efx) {}
static inline void efx_falcon_sriov_fini(struct efx_nic *efx) {}
int efx_siena_sriov_set_vf_mac(struct net_device *dev, int vf, u8 *mac);
int efx_siena_sriov_set_vf_vlan(struct net_device *dev, int vf,
u16 vlan, u8 qos);
int efx_siena_sriov_get_vf_config(struct net_device *dev, int vf,
struct ifla_vf_info *ivf);
int efx_siena_sriov_set_vf_spoofchk(struct net_device *net_dev, int vf,
bool spoofchk);
struct ethtool_ts_info;
int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel);
void efx_ptp_defer_probe_with_channel(struct efx_nic *efx);
......@@ -654,6 +564,7 @@ extern const struct efx_nic_type falcon_a1_nic_type;
extern const struct efx_nic_type falcon_b0_nic_type;
extern const struct efx_nic_type siena_a0_nic_type;
extern const struct efx_nic_type efx_hunt_a0_nic_type;
extern const struct efx_nic_type efx_hunt_a0_vf_nic_type;
/**************************************************************************
*
......
......@@ -25,6 +25,7 @@
#include "mcdi.h"
#include "mcdi_pcol.h"
#include "selftest.h"
#include "siena_sriov.h"
/* Hardware control for SFC9000 family including SFL9021 (aka Siena). */
......@@ -306,7 +307,9 @@ static int siena_probe_nic(struct efx_nic *efx)
if (rc)
goto fail5;
#ifdef CONFIG_SFC_SRIOV
efx_siena_sriov_probe(efx);
#endif
efx_ptp_defer_probe_with_channel(efx);
return 0;
......@@ -321,7 +324,8 @@ static int siena_probe_nic(struct efx_nic *efx)
return rc;
}
static void siena_rx_push_rss_config(struct efx_nic *efx)
static int siena_rx_push_rss_config(struct efx_nic *efx, bool user,
const u32 *rx_indir_table)
{
efx_oword_t temp;
......@@ -343,7 +347,11 @@ static void siena_rx_push_rss_config(struct efx_nic *efx)
FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH / 8);
efx_writeo(efx, &temp, FR_CZ_RX_RSS_IPV6_REG3);
memcpy(efx->rx_indir_table, rx_indir_table,
sizeof(efx->rx_indir_table));
efx_farch_rx_push_indir_table(efx);
return 0;
}
/* This call performs hardware-specific global initialisation, such as
......@@ -386,7 +394,7 @@ static int siena_init_nic(struct efx_nic *efx)
EFX_RX_USR_BUF_SIZE >> 5);
efx_writeo(efx, &temp, FR_AZ_RX_CFG);
siena_rx_push_rss_config(efx);
siena_rx_push_rss_config(efx, false, efx->rx_indir_table);
/* Enable event logging */
rc = efx_mcdi_log_ctrl(efx, true, false, 0);
......@@ -909,6 +917,8 @@ static int siena_mtd_probe(struct efx_nic *efx)
*/
const struct efx_nic_type siena_a0_nic_type = {
.is_vf = false,
.mem_bar = EFX_MEM_BAR,
.mem_map_size = siena_mem_map_size,
.probe = siena_probe_nic,
.remove = siena_remove_nic,
......@@ -996,11 +1006,22 @@ const struct efx_nic_type siena_a0_nic_type = {
#endif
.ptp_write_host_time = siena_ptp_write_host_time,
.ptp_set_ts_config = siena_ptp_set_ts_config,
#ifdef CONFIG_SFC_SRIOV
.sriov_configure = efx_siena_sriov_configure,
.sriov_init = efx_siena_sriov_init,
.sriov_fini = efx_siena_sriov_fini,
.sriov_mac_address_changed = efx_siena_sriov_mac_address_changed,
.sriov_wanted = efx_siena_sriov_wanted,
.sriov_reset = efx_siena_sriov_reset,
.sriov_flr = efx_siena_sriov_flr,
.sriov_set_vf_mac = efx_siena_sriov_set_vf_mac,
.sriov_set_vf_vlan = efx_siena_sriov_set_vf_vlan,
.sriov_set_vf_spoofchk = efx_siena_sriov_set_vf_spoofchk,
.sriov_get_vf_config = efx_siena_sriov_get_vf_config,
.vswitching_probe = efx_port_dummy_op_int,
.vswitching_restore = efx_port_dummy_op_int,
.vswitching_remove = efx_port_dummy_op_void,
#endif
.revision = EFX_REV_SIENA_A0,
.txd_ptr_tbl_base = FR_BZ_TX_DESC_PTR_TBL,
......
This diff is collapsed.
/****************************************************************************
* Driver for Solarflare network controllers and boards
* Copyright 2015 Solarflare Communications Inc.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation, incorporated herein by reference.
*/
#ifndef SIENA_SRIOV_H
#define SIENA_SRIOV_H
#include "net_driver.h"
/* On the SFC9000 family each port is associated with 1 PCI physical
* function (PF) handled by sfc and a configurable number of virtual
* functions (VFs) that may be handled by some other driver, often in
* a VM guest. The queue pointer registers are mapped in both PF and
* VF BARs such that an 8K region provides access to a single RX, TX
* and event queue (collectively a Virtual Interface, VI or VNIC).
*
* The PF has access to all 1024 VIs while VFs are mapped to VIs
* according to VI_BASE and VI_SCALE: VF i has access to VIs numbered
* in range [VI_BASE + i << VI_SCALE, VI_BASE + i + 1 << VI_SCALE).
* The number of VIs and the VI_SCALE value are configurable but must
* be established at boot time by firmware.
*/
/* Maximum VI_SCALE parameter supported by Siena */
#define EFX_VI_SCALE_MAX 6
/* Base VI to use for SR-IOV. Must be aligned to (1 << EFX_VI_SCALE_MAX),
* so this is the smallest allowed value.
*/
#define EFX_VI_BASE 128U
/* Maximum number of VFs allowed */
#define EFX_VF_COUNT_MAX 127
/* Limit EVQs on VFs to be only 8k to reduce buffer table reservation */
#define EFX_MAX_VF_EVQ_SIZE 8192UL
/* The number of buffer table entries reserved for each VI on a VF */
#define EFX_VF_BUFTBL_PER_VI \
((EFX_MAX_VF_EVQ_SIZE + 2 * EFX_MAX_DMAQ_SIZE) * \
sizeof(efx_qword_t) / EFX_BUF_SIZE)
int efx_siena_sriov_configure(struct efx_nic *efx, int num_vfs);
int efx_siena_sriov_init(struct efx_nic *efx);
void efx_siena_sriov_fini(struct efx_nic *efx);
void efx_siena_sriov_mac_address_changed(struct efx_nic *efx);
bool efx_siena_sriov_wanted(struct efx_nic *efx);
void efx_siena_sriov_reset(struct efx_nic *efx);
void efx_siena_sriov_flr(struct efx_nic *efx, unsigned flr);
int efx_siena_sriov_set_vf_mac(struct efx_nic *efx, int vf, u8 *mac);
int efx_siena_sriov_set_vf_vlan(struct efx_nic *efx, int vf,
u16 vlan, u8 qos);
int efx_siena_sriov_set_vf_spoofchk(struct efx_nic *efx, int vf,
bool spoofchk);
int efx_siena_sriov_get_vf_config(struct efx_nic *efx, int vf,
struct ifla_vf_info *ivf);
#ifdef CONFIG_SFC_SRIOV
static inline bool efx_siena_sriov_enabled(struct efx_nic *efx)
{
return efx->vf_init_count != 0;
}
#else /* !CONFIG_SFC_SRIOV */
static inline bool efx_siena_sriov_enabled(struct efx_nic *efx)
{
return false;
}
#endif /* CONFIG_SFC_SRIOV */
void efx_siena_sriov_probe(struct efx_nic *efx);
void efx_siena_sriov_tx_flush_done(struct efx_nic *efx, efx_qword_t *event);
void efx_siena_sriov_rx_flush_done(struct efx_nic *efx, efx_qword_t *event);
void efx_siena_sriov_event(struct efx_channel *channel, efx_qword_t *event);
void efx_siena_sriov_desc_fetch_err(struct efx_nic *efx, unsigned dmaq);
#endif /* SIENA_SRIOV_H */
/****************************************************************************
* Driver for Solarflare network controllers and boards
* Copyright 2014-2015 Solarflare Communications Inc.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation, incorporated herein by reference.
*/
#include <linux/module.h>
#include "net_driver.h"
#include "nic.h"
#include "sriov.h"
int efx_sriov_set_vf_mac(struct net_device *net_dev, int vf_i, u8 *mac)
{
struct efx_nic *efx = netdev_priv(net_dev);
if (efx->type->sriov_set_vf_mac)
return efx->type->sriov_set_vf_mac(efx, vf_i, mac);
else
return -EOPNOTSUPP;
}
int efx_sriov_set_vf_vlan(struct net_device *net_dev, int vf_i, u16 vlan,
u8 qos)
{
struct efx_nic *efx = netdev_priv(net_dev);
if (efx->type->sriov_set_vf_vlan) {
if ((vlan & ~VLAN_VID_MASK) ||
(qos & ~(VLAN_PRIO_MASK >> VLAN_PRIO_SHIFT)))
return -EINVAL;
return efx->type->sriov_set_vf_vlan(efx, vf_i, vlan, qos);
} else {
return -EOPNOTSUPP;
}
}
int efx_sriov_set_vf_spoofchk(struct net_device *net_dev, int vf_i,
bool spoofchk)
{
struct efx_nic *efx = netdev_priv(net_dev);
if (efx->type->sriov_set_vf_spoofchk)
return efx->type->sriov_set_vf_spoofchk(efx, vf_i, spoofchk);
else
return -EOPNOTSUPP;
}
int efx_sriov_get_vf_config(struct net_device *net_dev, int vf_i,
struct ifla_vf_info *ivi)
{
struct efx_nic *efx = netdev_priv(net_dev);
if (efx->type->sriov_get_vf_config)
return efx->type->sriov_get_vf_config(efx, vf_i, ivi);
else
return -EOPNOTSUPP;
}
/****************************************************************************
* Driver for Solarflare network controllers and boards
* Copyright 2014-2015 Solarflare Communications Inc.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation, incorporated herein by reference.
*/
#ifndef EFX_SRIOV_H
#define EFX_SRIOV_H
#include "net_driver.h"
#ifdef CONFIG_SFC_SRIOV
int efx_sriov_set_vf_mac(struct net_device *net_dev, int vf_i, u8 *mac);
int efx_sriov_set_vf_vlan(struct net_device *net_dev, int vf_i, u16 vlan,
u8 qos);
int efx_sriov_set_vf_spoofchk(struct net_device *net_dev, int vf_i,
bool spoofchk);
int efx_sriov_get_vf_config(struct net_device *net_dev, int vf_i,
struct ifla_vf_info *ivi);
#endif /* CONFIG_SFC_SRIOV */
#endif /* EFX_SRIOV_H */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment