Skip to content

L
linux
Commit 444cde13 authored by David S. Miller's avatar David S. Miller
Browse files
Options

Merge branch 'cpsw-xdp' 

Julien Panis says:

====================
Add minimal XDP support to TI AM65 CPSW Ethernet driver

This patch adds XDP support to TI AM65 CPSW Ethernet driver.

The following features are implemented: NETDEV_XDP_ACT_BASIC,
NETDEV_XDP_ACT_REDIRECT, and NETDEV_XDP_ACT_NDO_XMIT.

Zero-copy and non-linear XDP buffer supports are NOT implemented.

Besides, the page pool memory model is used to get better performance.
====================
Signed-off-by: default avatarJulien Panis <jpanis@baylibre.com>
parents 50aee97d 8acacc40
Show whitespace changes
Inline Side-by-side
Showing with 623 additions and 101 deletions
drivers/net/ethernet/ti/am65-cpsw-nuss.c
View file @ 444cde13
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
* *
*/ */
#include <linux/bpf_trace.h>
#include <linux/clk.h> #include <linux/clk.h>
#include <linux/etherdevice.h> #include <linux/etherdevice.h>
#include <linux/if_vlan.h> #include <linux/if_vlan.h>
...@@ -30,6 +31,7 @@ ...@@ -30,6 +31,7 @@
#include <linux/sys_soc.h> #include <linux/sys_soc.h>
#include <linux/dma/ti-cppi5.h> #include <linux/dma/ti-cppi5.h>
#include <linux/dma/k3-udma-glue.h> #include <linux/dma/k3-udma-glue.h>
#include <net/page_pool/helpers.h>
#include <net/switchdev.h> #include <net/switchdev.h>
#include "cpsw_ale.h" #include "cpsw_ale.h"
...@@ -138,6 +140,18 @@ ...@@ -138,6 +140,18 @@
#define AM65_CPSW_DEFAULT_TX_CHNS 8 #define AM65_CPSW_DEFAULT_TX_CHNS 8
/* CPPI streaming packet interface */
#define AM65_CPSW_CPPI_TX_FLOW_ID 0x3FFF
#define AM65_CPSW_CPPI_TX_PKT_TYPE 0x7
/* XDP */
#define AM65_CPSW_XDP_CONSUMED 2
#define AM65_CPSW_XDP_REDIRECT 1
#define AM65_CPSW_XDP_PASS 0
/* Include headroom compatible with both skb and xdpf */
#define AM65_CPSW_HEADROOM (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave, static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave,
const u8 *dev_addr) const u8 *dev_addr)
{ {
...@@ -305,12 +319,11 @@ static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev, ...@@ -305,12 +319,11 @@ static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev,
} }
static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common, static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common,
struct sk_buff *skb) struct page *page)
{ {
struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
struct cppi5_host_desc_t *desc_rx; struct cppi5_host_desc_t *desc_rx;
struct device *dev = common->dev; struct device *dev = common->dev;
u32 pkt_len = skb_tailroom(skb);
dma_addr_t desc_dma; dma_addr_t desc_dma;
dma_addr_t buf_dma; dma_addr_t buf_dma;
void *swdata; void *swdata;
...@@ -322,20 +335,22 @@ static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common, ...@@ -322,20 +335,22 @@ static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common,
} }
desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx); desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);
buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len, buf_dma = dma_map_single(rx_chn->dma_dev,
DMA_FROM_DEVICE); page_address(page) + AM65_CPSW_HEADROOM,
AM65_CPSW_MAX_PACKET_SIZE, DMA_FROM_DEVICE);
if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) { if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {
k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
dev_err(dev, "Failed to map rx skb buffer\n"); dev_err(dev, "Failed to map rx buffer\n");
return -EINVAL; return -EINVAL;
} }
cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT, cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
AM65_CPSW_NAV_PS_DATA_SIZE); AM65_CPSW_NAV_PS_DATA_SIZE);
k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma); k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);
cppi5_hdesc_attach_buf(desc_rx, buf_dma, skb_tailroom(skb), buf_dma, skb_tailroom(skb)); cppi5_hdesc_attach_buf(desc_rx, buf_dma, AM65_CPSW_MAX_PACKET_SIZE,
buf_dma, AM65_CPSW_MAX_PACKET_SIZE);
swdata = cppi5_hdesc_get_swdata(desc_rx); swdata = cppi5_hdesc_get_swdata(desc_rx);
*((void **)swdata) = skb; *((void **)swdata) = page_address(page);
return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0, desc_rx, desc_dma); return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0, desc_rx, desc_dma);
} }
...@@ -369,25 +384,131 @@ static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common); ...@@ -369,25 +384,131 @@ static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common);
static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port); static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port);
static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port); static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port);
static void am65_cpsw_destroy_xdp_rxqs(struct am65_cpsw_common *common)
{
struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
struct xdp_rxq_info *rxq;
int i;
for (i = 0; i < common->port_num; i++) {
rxq = &common->ports[i].xdp_rxq;
if (xdp_rxq_info_is_reg(rxq))
xdp_rxq_info_unreg(rxq);
}
if (rx_chn->page_pool) {
page_pool_destroy(rx_chn->page_pool);
rx_chn->page_pool = NULL;
}
}
static int am65_cpsw_create_xdp_rxqs(struct am65_cpsw_common *common)
{
struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
struct page_pool_params pp_params = {
.flags = PP_FLAG_DMA_MAP,
.order = 0,
.pool_size = AM65_CPSW_MAX_RX_DESC,
.nid = dev_to_node(common->dev),
.dev = common->dev,
.dma_dir = DMA_BIDIRECTIONAL,
.napi = &common->napi_rx,
};
struct xdp_rxq_info *rxq;
struct page_pool *pool;
int i, ret;
pool = page_pool_create(&pp_params);
if (IS_ERR(pool))
return PTR_ERR(pool);
rx_chn->page_pool = pool;
for (i = 0; i < common->port_num; i++) {
rxq = &common->ports[i].xdp_rxq;
ret = xdp_rxq_info_reg(rxq, common->ports[i].ndev, i, 0);
if (ret)
goto err;
ret = xdp_rxq_info_reg_mem_model(rxq, MEM_TYPE_PAGE_POOL, pool);
if (ret)
goto err;
}
return 0;
err:
am65_cpsw_destroy_xdp_rxqs(common);
return ret;
}
static int am65_cpsw_nuss_desc_idx(struct k3_cppi_desc_pool *desc_pool,
void *desc,
unsigned char dsize_log2)
{
void *pool_addr = k3_cppi_desc_pool_cpuaddr(desc_pool);
return (desc - pool_addr) >> dsize_log2;
}
static void am65_cpsw_nuss_set_buf_type(struct am65_cpsw_tx_chn *tx_chn,
struct cppi5_host_desc_t *desc,
enum am65_cpsw_tx_buf_type buf_type)
{
int desc_idx;
desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc,
tx_chn->dsize_log2);
k3_cppi_desc_pool_desc_info_set(tx_chn->desc_pool, desc_idx,
(void *)buf_type);
}
static enum am65_cpsw_tx_buf_type am65_cpsw_nuss_buf_type(struct am65_cpsw_tx_chn *tx_chn,
dma_addr_t desc_dma)
{
struct cppi5_host_desc_t *desc_tx;
int desc_idx;
desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
desc_idx = am65_cpsw_nuss_desc_idx(tx_chn->desc_pool, desc_tx,
tx_chn->dsize_log2);
return (enum am65_cpsw_tx_buf_type)k3_cppi_desc_pool_desc_info(tx_chn->desc_pool,
desc_idx);
}
static inline void am65_cpsw_put_page(struct am65_cpsw_rx_chn *rx_chn,
struct page *page,
bool allow_direct,
int desc_idx)
{
page_pool_put_full_page(rx_chn->page_pool, page, allow_direct);
rx_chn->pages[desc_idx] = NULL;
}
static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma) static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
{ {
struct am65_cpsw_rx_chn *rx_chn = data; struct am65_cpsw_rx_chn *rx_chn = data;
struct cppi5_host_desc_t *desc_rx; struct cppi5_host_desc_t *desc_rx;
struct sk_buff *skb;
dma_addr_t buf_dma; dma_addr_t buf_dma;
u32 buf_dma_len; u32 buf_dma_len;
void *page_addr;
void **swdata; void **swdata;
int desc_idx;
desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
swdata = cppi5_hdesc_get_swdata(desc_rx); swdata = cppi5_hdesc_get_swdata(desc_rx);
skb = *swdata; page_addr = *swdata;
cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE); dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
dev_kfree_skb_any(skb); desc_idx = am65_cpsw_nuss_desc_idx(rx_chn->desc_pool, desc_rx,
rx_chn->dsize_log2);
am65_cpsw_put_page(rx_chn, virt_to_page(page_addr), false, desc_idx);
} }
static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn, static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
...@@ -440,12 +561,32 @@ static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma) ...@@ -440,12 +561,32 @@ static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
dev_kfree_skb_any(skb); dev_kfree_skb_any(skb);
} }
static struct sk_buff *am65_cpsw_build_skb(void *page_addr,
struct net_device *ndev,
unsigned int len)
{
struct sk_buff *skb;
len += AM65_CPSW_HEADROOM;
skb = build_skb(page_addr, len);
if (unlikely(!skb))
return NULL;
skb_reserve(skb, AM65_CPSW_HEADROOM);
skb->dev = ndev;
return skb;
}
static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
{ {
struct am65_cpsw_host *host_p = am65_common_get_host(common); struct am65_cpsw_host *host_p = am65_common_get_host(common);
struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
struct am65_cpsw_tx_chn *tx_chn = common->tx_chns;
int port_idx, i, ret, tx; int port_idx, i, ret, tx;
struct sk_buff *skb;
u32 val, port_mask; u32 val, port_mask;
struct page *page;
if (common->usage_count) if (common->usage_count)
return 0; return 0;
...@@ -505,25 +646,29 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) ...@@ -505,25 +646,29 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
am65_cpsw_qos_tx_p0_rate_init(common); am65_cpsw_qos_tx_p0_rate_init(common);
for (i = 0; i < common->rx_chns.descs_num; i++) { ret = am65_cpsw_create_xdp_rxqs(common);
skb = __netdev_alloc_skb_ip_align(NULL, if (ret) {
AM65_CPSW_MAX_PACKET_SIZE, dev_err(common->dev, "Failed to create XDP rx queues\n");
GFP_KERNEL); return ret;
if (!skb) { }
for (i = 0; i < rx_chn->descs_num; i++) {
page = page_pool_dev_alloc_pages(rx_chn->page_pool);
if (!page) {
ret = -ENOMEM; ret = -ENOMEM;
dev_err(common->dev, "cannot allocate skb\n");
if (i) if (i)
goto fail_rx; goto fail_rx;
return ret; return ret;
} }
rx_chn->pages[i] = page;
ret = am65_cpsw_nuss_rx_push(common, skb); ret = am65_cpsw_nuss_rx_push(common, page);
if (ret < 0) { if (ret < 0) {
dev_err(common->dev, dev_err(common->dev,
"cannot submit skb to channel rx, error %d\n", "cannot submit page to channel rx: %d\n",
ret); ret);
kfree_skb(skb); am65_cpsw_put_page(rx_chn, page, false, i);
if (i) if (i)
goto fail_rx; goto fail_rx;
...@@ -531,27 +676,27 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) ...@@ -531,27 +676,27 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
} }
} }
ret = k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn); ret = k3_udma_glue_enable_rx_chn(rx_chn->rx_chn);
if (ret) { if (ret) {
dev_err(common->dev, "couldn't enable rx chn: %d\n", ret); dev_err(common->dev, "couldn't enable rx chn: %d\n", ret);
goto fail_rx; goto fail_rx;
} }
for (tx = 0; tx < common->tx_ch_num; tx++) { for (tx = 0; tx < common->tx_ch_num; tx++) {
ret = k3_udma_glue_enable_tx_chn(common->tx_chns[tx].tx_chn); ret = k3_udma_glue_enable_tx_chn(tx_chn[tx].tx_chn);
if (ret) { if (ret) {
dev_err(common->dev, "couldn't enable tx chn %d: %d\n", dev_err(common->dev, "couldn't enable tx chn %d: %d\n",
tx, ret); tx, ret);
tx--; tx--;
goto fail_tx; goto fail_tx;
} }
napi_enable(&common->tx_chns[tx].napi_tx); napi_enable(&tx_chn[tx].napi_tx);
} }
napi_enable(&common->napi_rx); napi_enable(&common->napi_rx);
if (common->rx_irq_disabled) { if (common->rx_irq_disabled) {
common->rx_irq_disabled = false; common->rx_irq_disabled = false;
enable_irq(common->rx_chns.irq); enable_irq(rx_chn->irq);
} }
dev_dbg(common->dev, "cpsw_nuss started\n"); dev_dbg(common->dev, "cpsw_nuss started\n");
...@@ -559,22 +704,23 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) ...@@ -559,22 +704,23 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
fail_tx: fail_tx:
while (tx >= 0) { while (tx >= 0) {
napi_disable(&common->tx_chns[tx].napi_tx); napi_disable(&tx_chn[tx].napi_tx);
k3_udma_glue_disable_tx_chn(common->tx_chns[tx].tx_chn); k3_udma_glue_disable_tx_chn(tx_chn[tx].tx_chn);
tx--; tx--;
} }
k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn); k3_udma_glue_disable_rx_chn(rx_chn->rx_chn);
fail_rx: fail_rx:
k3_udma_glue_reset_rx_chn(common->rx_chns.rx_chn, 0, k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, 0, rx_chn,
&common->rx_chns,
am65_cpsw_nuss_rx_cleanup, 0); am65_cpsw_nuss_rx_cleanup, 0);
return ret; return ret;
} }
static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common) static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
{ {
struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
struct am65_cpsw_tx_chn *tx_chn = common->tx_chns;
int i; int i;
if (common->usage_count != 1) if (common->usage_count != 1)
...@@ -590,26 +736,25 @@ static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common) ...@@ -590,26 +736,25 @@ static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
reinit_completion(&common->tdown_complete); reinit_completion(&common->tdown_complete);
for (i = 0; i < common->tx_ch_num; i++) for (i = 0; i < common->tx_ch_num; i++)
k3_udma_glue_tdown_tx_chn(common->tx_chns[i].tx_chn, false); k3_udma_glue_tdown_tx_chn(tx_chn[i].tx_chn, false);
i = wait_for_completion_timeout(&common->tdown_complete, i = wait_for_completion_timeout(&common->tdown_complete,
msecs_to_jiffies(1000)); msecs_to_jiffies(1000));
if (!i) if (!i)
dev_err(common->dev, "tx timeout\n"); dev_err(common->dev, "tx timeout\n");
for (i = 0; i < common->tx_ch_num; i++) { for (i = 0; i < common->tx_ch_num; i++) {
napi_disable(&common->tx_chns[i].napi_tx); napi_disable(&tx_chn[i].napi_tx);
hrtimer_cancel(&common->tx_chns[i].tx_hrtimer); hrtimer_cancel(&tx_chn[i].tx_hrtimer);
} }
for (i = 0; i < common->tx_ch_num; i++) { for (i = 0; i < common->tx_ch_num; i++) {
k3_udma_glue_reset_tx_chn(common->tx_chns[i].tx_chn, k3_udma_glue_reset_tx_chn(tx_chn[i].tx_chn, &tx_chn[i],
&common->tx_chns[i],
am65_cpsw_nuss_tx_cleanup); am65_cpsw_nuss_tx_cleanup);
k3_udma_glue_disable_tx_chn(common->tx_chns[i].tx_chn); k3_udma_glue_disable_tx_chn(tx_chn[i].tx_chn);
} }
reinit_completion(&common->tdown_complete); reinit_completion(&common->tdown_complete);
k3_udma_glue_tdown_rx_chn(common->rx_chns.rx_chn, true); k3_udma_glue_tdown_rx_chn(rx_chn->rx_chn, true);
if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) { if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) {
i = wait_for_completion_timeout(&common->tdown_complete, msecs_to_jiffies(1000)); i = wait_for_completion_timeout(&common->tdown_complete, msecs_to_jiffies(1000));
...@@ -621,17 +766,22 @@ static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common) ...@@ -621,17 +766,22 @@ static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
hrtimer_cancel(&common->rx_hrtimer); hrtimer_cancel(&common->rx_hrtimer);
for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++) for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++)
k3_udma_glue_reset_rx_chn(common->rx_chns.rx_chn, i, k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, i, rx_chn,
&common->rx_chns,
am65_cpsw_nuss_rx_cleanup, !!i); am65_cpsw_nuss_rx_cleanup, !!i);
k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn); k3_udma_glue_disable_rx_chn(rx_chn->rx_chn);
cpsw_ale_stop(common->ale); cpsw_ale_stop(common->ale);
writel(0, common->cpsw_base + AM65_CPSW_REG_CTL); writel(0, common->cpsw_base + AM65_CPSW_REG_CTL);
writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN); writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
for (i = 0; i < rx_chn->descs_num; i++) {
if (rx_chn->pages[i])
am65_cpsw_put_page(rx_chn, rx_chn->pages[i], false, i);
}
am65_cpsw_destroy_xdp_rxqs(common);
dev_dbg(common->dev, "cpsw_nuss stopped\n"); dev_dbg(common->dev, "cpsw_nuss stopped\n");
return 0; return 0;
} }
...@@ -749,6 +899,151 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev) ...@@ -749,6 +899,151 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
return ret; return ret;
} }
static int am65_cpsw_xdp_tx_frame(struct net_device *ndev,
struct am65_cpsw_tx_chn *tx_chn,
struct xdp_frame *xdpf,
enum am65_cpsw_tx_buf_type buf_type)
{
struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
struct cppi5_host_desc_t *host_desc;
struct netdev_queue *netif_txq;
dma_addr_t dma_desc, dma_buf;
u32 pkt_len = xdpf->len;
void **swdata;
int ret;
host_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
if (unlikely(!host_desc)) {
ndev->stats.tx_dropped++;
return -ENOMEM;
}
am65_cpsw_nuss_set_buf_type(tx_chn, host_desc, buf_type);
dma_buf = dma_map_single(tx_chn->dma_dev, xdpf->data,
pkt_len, DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(tx_chn->dma_dev, dma_buf))) {
ndev->stats.tx_dropped++;
ret = -ENOMEM;
goto pool_free;
}
cppi5_hdesc_init(host_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
AM65_CPSW_NAV_PS_DATA_SIZE);
cppi5_hdesc_set_pkttype(host_desc, AM65_CPSW_CPPI_TX_PKT_TYPE);
cppi5_hdesc_set_pktlen(host_desc, pkt_len);
cppi5_desc_set_pktids(&host_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID);
cppi5_desc_set_tags_ids(&host_desc->hdr, 0, port->port_id);
k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &dma_buf);
cppi5_hdesc_attach_buf(host_desc, dma_buf, pkt_len, dma_buf, pkt_len);
swdata = cppi5_hdesc_get_swdata(host_desc);
*(swdata) = xdpf;
/* Report BQL before sending the packet */
netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
netdev_tx_sent_queue(netif_txq, pkt_len);
dma_desc = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, host_desc);
if (AM65_CPSW_IS_CPSW2G(common)) {
ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc,
dma_desc);
} else {
spin_lock_bh(&tx_chn->lock);
ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, host_desc,
dma_desc);
spin_unlock_bh(&tx_chn->lock);
}
if (ret) {
/* Inform BQL */
netdev_tx_completed_queue(netif_txq, 1, pkt_len);
ndev->stats.tx_errors++;
goto dma_unmap;
}
return 0;
dma_unmap:
k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &dma_buf);
dma_unmap_single(tx_chn->dma_dev, dma_buf, pkt_len, DMA_TO_DEVICE);
pool_free:
k3_cppi_desc_pool_free(tx_chn->desc_pool, host_desc);
return ret;
}
static int am65_cpsw_run_xdp(struct am65_cpsw_common *common,
struct am65_cpsw_port *port,
struct xdp_buff *xdp,
int desc_idx, int cpu, int *len)
{
struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
struct net_device *ndev = port->ndev;
int ret = AM65_CPSW_XDP_CONSUMED;
struct am65_cpsw_tx_chn *tx_chn;
struct netdev_queue *netif_txq;
struct xdp_frame *xdpf;
struct bpf_prog *prog;
struct page *page;
u32 act;
prog = READ_ONCE(port->xdp_prog);
if (!prog)
return AM65_CPSW_XDP_PASS;
act = bpf_prog_run_xdp(prog, xdp);
/* XDP prog might have changed packet data and boundaries */
*len = xdp->data_end - xdp->data;
switch (act) {
case XDP_PASS:
ret = AM65_CPSW_XDP_PASS;
goto out;
case XDP_TX:
tx_chn = &common->tx_chns[cpu % AM65_CPSW_MAX_TX_QUEUES];
netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
xdpf = xdp_convert_buff_to_frame(xdp);
if (unlikely(!xdpf))
break;
__netif_tx_lock(netif_txq, cpu);
ret = am65_cpsw_xdp_tx_frame(ndev, tx_chn, xdpf,
AM65_CPSW_TX_BUF_TYPE_XDP_TX);
__netif_tx_unlock(netif_txq);
if (ret)
break;
ndev->stats.rx_bytes += *len;
ndev->stats.rx_packets++;
ret = AM65_CPSW_XDP_CONSUMED;
goto out;
case XDP_REDIRECT:
if (unlikely(xdp_do_redirect(ndev, xdp, prog)))
break;
ndev->stats.rx_bytes += *len;
ndev->stats.rx_packets++;
ret = AM65_CPSW_XDP_REDIRECT;
goto out;
default:
bpf_warn_invalid_xdp_action(ndev, prog, act);
fallthrough;
case XDP_ABORTED:
trace_xdp_exception(ndev, prog, act);
fallthrough;
case XDP_DROP:
ndev->stats.rx_dropped++;
}
page = virt_to_head_page(xdp->data);
am65_cpsw_put_page(rx_chn, page, true, desc_idx);
out:
return ret;
}
static void am65_cpsw_nuss_rx_ts(struct sk_buff *skb, u32 *psdata) static void am65_cpsw_nuss_rx_ts(struct sk_buff *skb, u32 *psdata)
{ {
struct skb_shared_hwtstamps *ssh; struct skb_shared_hwtstamps *ssh;
...@@ -795,7 +1090,7 @@ static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info) ...@@ -795,7 +1090,7 @@ static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info)
} }
static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common, static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
u32 flow_idx) u32 flow_idx, int cpu)
{ {
struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns; struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
u32 buf_dma_len, pkt_len, port_id = 0, csum_info; u32 buf_dma_len, pkt_len, port_id = 0, csum_info;
...@@ -803,13 +1098,16 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common, ...@@ -803,13 +1098,16 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
struct am65_cpsw_ndev_stats *stats; struct am65_cpsw_ndev_stats *stats;
struct cppi5_host_desc_t *desc_rx; struct cppi5_host_desc_t *desc_rx;
struct device *dev = common->dev; struct device *dev = common->dev;
struct sk_buff *skb, *new_skb; struct page *page, *new_page;
dma_addr_t desc_dma, buf_dma; dma_addr_t desc_dma, buf_dma;
struct am65_cpsw_port *port; struct am65_cpsw_port *port;
int headroom, desc_idx, ret;
struct net_device *ndev; struct net_device *ndev;
struct sk_buff *skb;
struct xdp_buff xdp;
void *page_addr;
void **swdata; void **swdata;
u32 *psdata; u32 *psdata;
int ret = 0;
ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma); ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);
if (ret) { if (ret) {
...@@ -830,7 +1128,8 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common, ...@@ -830,7 +1128,8 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
__func__, flow_idx, &desc_dma); __func__, flow_idx, &desc_dma);
swdata = cppi5_hdesc_get_swdata(desc_rx); swdata = cppi5_hdesc_get_swdata(desc_rx);
skb = *swdata; page_addr = *swdata;
page = virt_to_page(page_addr);
cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
pkt_len = cppi5_hdesc_get_pktlen(desc_rx); pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
...@@ -838,12 +1137,7 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common, ...@@ -838,12 +1137,7 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id); dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id);
port = am65_common_get_port(common, port_id); port = am65_common_get_port(common, port_id);
ndev = port->ndev; ndev = port->ndev;
skb->dev = ndev;
psdata = cppi5_hdesc_get_psdata(desc_rx); psdata = cppi5_hdesc_get_psdata(desc_rx);
/* add RX timestamp */
if (port->rx_ts_enabled)
am65_cpsw_nuss_rx_ts(skb, psdata);
csum_info = psdata[2]; csum_info = psdata[2];
dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info); dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info);
...@@ -851,11 +1145,40 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common, ...@@ -851,11 +1145,40 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
new_skb = netdev_alloc_skb_ip_align(ndev, AM65_CPSW_MAX_PACKET_SIZE); desc_idx = am65_cpsw_nuss_desc_idx(rx_chn->desc_pool, desc_rx,
if (new_skb) { rx_chn->dsize_log2);
skb = am65_cpsw_build_skb(page_addr, ndev,
AM65_CPSW_MAX_PACKET_SIZE);
if (unlikely(!skb)) {
new_page = page;
goto requeue;
}
if (port->xdp_prog) {
xdp_init_buff(&xdp, AM65_CPSW_MAX_PACKET_SIZE, &port->xdp_rxq);
xdp_prepare_buff(&xdp, page_addr, skb_headroom(skb),
pkt_len, false);
ret = am65_cpsw_run_xdp(common, port, &xdp, desc_idx,
cpu, &pkt_len);
if (ret != AM65_CPSW_XDP_PASS)
return ret;
/* Compute additional headroom to be reserved */
headroom = (xdp.data - xdp.data_hard_start) - skb_headroom(skb);
skb_reserve(skb, headroom);
}
/* Pass skb to netstack if no XDP prog or returned XDP_PASS */
if (port->rx_ts_enabled)
am65_cpsw_nuss_rx_ts(skb, psdata);
ndev_priv = netdev_priv(ndev); ndev_priv = netdev_priv(ndev);
am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark); am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark);
skb_put(skb, pkt_len); skb_put(skb, pkt_len);
skb_mark_for_recycle(skb);
skb->protocol = eth_type_trans(skb, ndev); skb->protocol = eth_type_trans(skb, ndev);
am65_cpsw_nuss_rx_csum(skb, csum_info); am65_cpsw_nuss_rx_csum(skb, csum_info);
napi_gro_receive(&common->napi_rx, skb); napi_gro_receive(&common->napi_rx, skb);
...@@ -866,21 +1189,22 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common, ...@@ -866,21 +1189,22 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
stats->rx_packets++; stats->rx_packets++;
stats->rx_bytes += pkt_len; stats->rx_bytes += pkt_len;
u64_stats_update_end(&stats->syncp); u64_stats_update_end(&stats->syncp);
kmemleak_not_leak(new_skb);
} else { new_page = page_pool_dev_alloc_pages(rx_chn->page_pool);
ndev->stats.rx_dropped++; if (unlikely(!new_page))
new_skb = skb; return -ENOMEM;
} rx_chn->pages[desc_idx] = new_page;
if (netif_dormant(ndev)) { if (netif_dormant(ndev)) {
dev_kfree_skb_any(new_skb); am65_cpsw_put_page(rx_chn, new_page, true, desc_idx);
ndev->stats.rx_dropped++; ndev->stats.rx_dropped++;
return 0; return 0;
} }
ret = am65_cpsw_nuss_rx_push(common, new_skb); requeue:
ret = am65_cpsw_nuss_rx_push(common, new_page);
if (WARN_ON(ret < 0)) { if (WARN_ON(ret < 0)) {
dev_kfree_skb_any(new_skb); am65_cpsw_put_page(rx_chn, new_page, true, desc_idx);
ndev->stats.rx_errors++; ndev->stats.rx_errors++;
ndev->stats.rx_dropped++; ndev->stats.rx_dropped++;
} }
...@@ -901,6 +1225,8 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget) ...@@ -901,6 +1225,8 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
{ {
struct am65_cpsw_common *common = am65_cpsw_napi_to_common(napi_rx); struct am65_cpsw_common *common = am65_cpsw_napi_to_common(napi_rx);
int flow = AM65_CPSW_MAX_RX_FLOWS; int flow = AM65_CPSW_MAX_RX_FLOWS;
int cpu = smp_processor_id();
bool xdp_redirect = false;
int cur_budget, ret; int cur_budget, ret;
int num_rx = 0; int num_rx = 0;
...@@ -909,9 +1235,12 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget) ...@@ -909,9 +1235,12 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
cur_budget = budget - num_rx; cur_budget = budget - num_rx;
while (cur_budget--) { while (cur_budget--) {
ret = am65_cpsw_nuss_rx_packets(common, flow); ret = am65_cpsw_nuss_rx_packets(common, flow, cpu);
if (ret) if (ret) {
if (ret == AM65_CPSW_XDP_REDIRECT)
xdp_redirect = true;
break; break;
}
num_rx++; num_rx++;
} }
...@@ -919,6 +1248,9 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget) ...@@ -919,6 +1248,9 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
break; break;
} }
if (xdp_redirect)
xdp_do_flush();
dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget); dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);
if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) { if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) {
...@@ -938,7 +1270,7 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget) ...@@ -938,7 +1270,7 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
} }
static struct sk_buff * static struct sk_buff *
am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn, am65_cpsw_nuss_tx_compl_packet_skb(struct am65_cpsw_tx_chn *tx_chn,
dma_addr_t desc_dma) dma_addr_t desc_dma)
{ {
struct am65_cpsw_ndev_priv *ndev_priv; struct am65_cpsw_ndev_priv *ndev_priv;
...@@ -968,6 +1300,39 @@ am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn, ...@@ -968,6 +1300,39 @@ am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn,
return skb; return skb;
} }
static struct xdp_frame *
am65_cpsw_nuss_tx_compl_packet_xdp(struct am65_cpsw_common *common,
struct am65_cpsw_tx_chn *tx_chn,
dma_addr_t desc_dma,
struct net_device **ndev)
{
struct am65_cpsw_ndev_priv *ndev_priv;
struct am65_cpsw_ndev_stats *stats;
struct cppi5_host_desc_t *desc_tx;
struct am65_cpsw_port *port;
struct xdp_frame *xdpf;
u32 port_id = 0;
void **swdata;
desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
cppi5_desc_get_tags_ids(&desc_tx->hdr, NULL, &port_id);
swdata = cppi5_hdesc_get_swdata(desc_tx);
xdpf = *(swdata);
am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
port = am65_common_get_port(common, port_id);
*ndev = port->ndev;
ndev_priv = netdev_priv(*ndev);
stats = this_cpu_ptr(ndev_priv->stats);
u64_stats_update_begin(&stats->syncp);
stats->tx_packets++;
stats->tx_bytes += xdpf->len;
u64_stats_update_end(&stats->syncp);
return xdpf;
}
static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev, static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,
struct netdev_queue *netif_txq) struct netdev_queue *netif_txq)
{ {
...@@ -988,11 +1353,13 @@ static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_d ...@@ -988,11 +1353,13 @@ static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_d
static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common, static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
int chn, unsigned int budget, bool *tdown) int chn, unsigned int budget, bool *tdown)
{ {
enum am65_cpsw_tx_buf_type buf_type;
struct device *dev = common->dev; struct device *dev = common->dev;
struct am65_cpsw_tx_chn *tx_chn; struct am65_cpsw_tx_chn *tx_chn;
struct netdev_queue *netif_txq; struct netdev_queue *netif_txq;
unsigned int total_bytes = 0; unsigned int total_bytes = 0;
struct net_device *ndev; struct net_device *ndev;
struct xdp_frame *xdpf;
struct sk_buff *skb; struct sk_buff *skb;
dma_addr_t desc_dma; dma_addr_t desc_dma;
int res, num_tx = 0; int res, num_tx = 0;
...@@ -1013,10 +1380,21 @@ static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common, ...@@ -1013,10 +1380,21 @@ static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
break; break;
} }
skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma); buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);
total_bytes = skb->len; if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) {
skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma);
ndev = skb->dev; ndev = skb->dev;
total_bytes = skb->len;
napi_consume_skb(skb, budget); napi_consume_skb(skb, budget);
} else {
xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn,
desc_dma, &ndev);
total_bytes = xdpf->len;
if (buf_type == AM65_CPSW_TX_BUF_TYPE_XDP_TX)
xdp_return_frame_rx_napi(xdpf);
else
xdp_return_frame(xdpf);
}
num_tx++; num_tx++;
netif_txq = netdev_get_tx_queue(ndev, chn); netif_txq = netdev_get_tx_queue(ndev, chn);
...@@ -1034,11 +1412,13 @@ static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common, ...@@ -1034,11 +1412,13 @@ static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common, static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common,
int chn, unsigned int budget, bool *tdown) int chn, unsigned int budget, bool *tdown)
{ {
enum am65_cpsw_tx_buf_type buf_type;
struct device *dev = common->dev; struct device *dev = common->dev;
struct am65_cpsw_tx_chn *tx_chn; struct am65_cpsw_tx_chn *tx_chn;
struct netdev_queue *netif_txq; struct netdev_queue *netif_txq;
unsigned int total_bytes = 0; unsigned int total_bytes = 0;
struct net_device *ndev; struct net_device *ndev;
struct xdp_frame *xdpf;
struct sk_buff *skb; struct sk_buff *skb;
dma_addr_t desc_dma; dma_addr_t desc_dma;
int res, num_tx = 0; int res, num_tx = 0;
...@@ -1057,11 +1437,21 @@ static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common, ...@@ -1057,11 +1437,21 @@ static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common,
break; break;
} }
skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma); buf_type = am65_cpsw_nuss_buf_type(tx_chn, desc_dma);
if (buf_type == AM65_CPSW_TX_BUF_TYPE_SKB) {
skb = am65_cpsw_nuss_tx_compl_packet_skb(tx_chn, desc_dma);
ndev = skb->dev; ndev = skb->dev;
total_bytes += skb->len; total_bytes += skb->len;
napi_consume_skb(skb, budget); napi_consume_skb(skb, budget);
} else {
xdpf = am65_cpsw_nuss_tx_compl_packet_xdp(common, tx_chn,
desc_dma, &ndev);
total_bytes += xdpf->len;
if (buf_type == AM65_CPSW_TX_BUF_TYPE_XDP_TX)
xdp_return_frame_rx_napi(xdpf);
else
xdp_return_frame(xdpf);
}
num_tx++; num_tx++;
} }
...@@ -1183,10 +1573,13 @@ static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb, ...@@ -1183,10 +1573,13 @@ static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
goto busy_stop_q; goto busy_stop_q;
} }
am65_cpsw_nuss_set_buf_type(tx_chn, first_desc,
AM65_CPSW_TX_BUF_TYPE_SKB);
cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT, cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
AM65_CPSW_NAV_PS_DATA_SIZE); AM65_CPSW_NAV_PS_DATA_SIZE);
cppi5_desc_set_pktids(&first_desc->hdr, 0, 0x3FFF); cppi5_desc_set_pktids(&first_desc->hdr, 0, AM65_CPSW_CPPI_TX_FLOW_ID);
cppi5_hdesc_set_pkttype(first_desc, 0x7); cppi5_hdesc_set_pkttype(first_desc, AM65_CPSW_CPPI_TX_PKT_TYPE);
cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id); cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id);
k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
...@@ -1225,6 +1618,9 @@ static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb, ...@@ -1225,6 +1618,9 @@ static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
goto busy_free_descs; goto busy_free_descs;
} }
am65_cpsw_nuss_set_buf_type(tx_chn, next_desc,
AM65_CPSW_TX_BUF_TYPE_SKB);
buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size, buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,
DMA_TO_DEVICE); DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) { if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
...@@ -1488,6 +1884,59 @@ static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev, ...@@ -1488,6 +1884,59 @@ static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev,
stats->tx_dropped = dev->stats.tx_dropped; stats->tx_dropped = dev->stats.tx_dropped;
} }
static int am65_cpsw_xdp_prog_setup(struct net_device *ndev,
struct bpf_prog *prog)
{
struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
bool running = netif_running(ndev);
struct bpf_prog *old_prog;
if (running)
am65_cpsw_nuss_ndo_slave_stop(ndev);
old_prog = xchg(&port->xdp_prog, prog);
if (old_prog)
bpf_prog_put(old_prog);
if (running)
return am65_cpsw_nuss_ndo_slave_open(ndev);
return 0;
}
static int am65_cpsw_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
{
switch (bpf->command) {
case XDP_SETUP_PROG:
return am65_cpsw_xdp_prog_setup(ndev, bpf->prog);
default:
return -EINVAL;
}
}
static int am65_cpsw_ndo_xdp_xmit(struct net_device *ndev, int n,
struct xdp_frame **frames, u32 flags)
{
struct am65_cpsw_tx_chn *tx_chn;
struct netdev_queue *netif_txq;
int cpu = smp_processor_id();
int i, nxmit = 0;
tx_chn = &am65_ndev_to_common(ndev)->tx_chns[cpu % AM65_CPSW_MAX_TX_QUEUES];
netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
__netif_tx_lock(netif_txq, cpu);
for (i = 0; i < n; i++) {
if (am65_cpsw_xdp_tx_frame(ndev, tx_chn, frames[i],
AM65_CPSW_TX_BUF_TYPE_XDP_NDO))
break;
nxmit++;
}
__netif_tx_unlock(netif_txq);
return nxmit;
}
static const struct net_device_ops am65_cpsw_nuss_netdev_ops = { static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
.ndo_open = am65_cpsw_nuss_ndo_slave_open, .ndo_open = am65_cpsw_nuss_ndo_slave_open,
.ndo_stop = am65_cpsw_nuss_ndo_slave_stop, .ndo_stop = am65_cpsw_nuss_ndo_slave_stop,
...@@ -1502,6 +1951,8 @@ static const struct net_device_ops am65_cpsw_nuss_netdev_ops = { ...@@ -1502,6 +1951,8 @@ static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
.ndo_eth_ioctl = am65_cpsw_nuss_ndo_slave_ioctl, .ndo_eth_ioctl = am65_cpsw_nuss_ndo_slave_ioctl,
.ndo_setup_tc = am65_cpsw_qos_ndo_setup_tc, .ndo_setup_tc = am65_cpsw_qos_ndo_setup_tc,
.ndo_set_tx_maxrate = am65_cpsw_qos_ndo_tx_p0_set_maxrate, .ndo_set_tx_maxrate = am65_cpsw_qos_ndo_tx_p0_set_maxrate,
.ndo_bpf = am65_cpsw_ndo_bpf,
.ndo_xdp_xmit = am65_cpsw_ndo_xdp_xmit,
}; };
static void am65_cpsw_disable_phy(struct phy *phy) static void am65_cpsw_disable_phy(struct phy *phy)
...@@ -1772,7 +2223,7 @@ static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common) ...@@ -1772,7 +2223,7 @@ static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common)
.mode = K3_RINGACC_RING_MODE_RING, .mode = K3_RINGACC_RING_MODE_RING,
.flags = 0 .flags = 0
}; };
u32 hdesc_size; u32 hdesc_size, hdesc_size_out;
int i, ret = 0; int i, ret = 0;
hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE, hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
...@@ -1816,6 +2267,10 @@ static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common) ...@@ -1816,6 +2267,10 @@ static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common)
goto err; goto err;
} }
hdesc_size_out = k3_cppi_desc_pool_desc_size(tx_chn->desc_pool);
tx_chn->dsize_log2 = __fls(hdesc_size_out);
WARN_ON(hdesc_size_out != (1 << tx_chn->dsize_log2));
tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn); tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);
if (tx_chn->irq < 0) { if (tx_chn->irq < 0) {
dev_err(dev, "Failed to get tx dma irq %d\n", dev_err(dev, "Failed to get tx dma irq %d\n",
...@@ -1862,8 +2317,8 @@ static void am65_cpsw_nuss_free_rx_chns(void *data) ...@@ -1862,8 +2317,8 @@ static void am65_cpsw_nuss_free_rx_chns(void *data)
static void am65_cpsw_nuss_remove_rx_chns(void *data) static void am65_cpsw_nuss_remove_rx_chns(void *data)
{ {
struct am65_cpsw_common *common = data; struct am65_cpsw_common *common = data;
struct am65_cpsw_rx_chn *rx_chn;
struct device *dev = common->dev; struct device *dev = common->dev;
struct am65_cpsw_rx_chn *rx_chn;
rx_chn = &common->rx_chns; rx_chn = &common->rx_chns;
devm_remove_action(dev, am65_cpsw_nuss_free_rx_chns, common); devm_remove_action(dev, am65_cpsw_nuss_free_rx_chns, common);
...@@ -1873,11 +2328,7 @@ static void am65_cpsw_nuss_remove_rx_chns(void *data) ...@@ -1873,11 +2328,7 @@ static void am65_cpsw_nuss_remove_rx_chns(void *data)
netif_napi_del(&common->napi_rx); netif_napi_del(&common->napi_rx);
if (!IS_ERR_OR_NULL(rx_chn->desc_pool)) am65_cpsw_nuss_free_rx_chns(common);
k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
if (!IS_ERR_OR_NULL(rx_chn->rx_chn))
k3_udma_glue_release_rx_chn(rx_chn->rx_chn);
common->rx_flow_id_base = -1; common->rx_flow_id_base = -1;
} }
...@@ -1888,7 +2339,7 @@ static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common) ...@@ -1888,7 +2339,7 @@ static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common)
struct k3_udma_glue_rx_channel_cfg rx_cfg = { 0 }; struct k3_udma_glue_rx_channel_cfg rx_cfg = { 0 };
u32 max_desc_num = AM65_CPSW_MAX_RX_DESC; u32 max_desc_num = AM65_CPSW_MAX_RX_DESC;
struct device *dev = common->dev; struct device *dev = common->dev;
u32 hdesc_size; u32 hdesc_size, hdesc_size_out;
u32 fdqring_id; u32 fdqring_id;
int i, ret = 0; int i, ret = 0;
...@@ -1920,6 +2371,17 @@ static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common) ...@@ -1920,6 +2371,17 @@ static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common)
goto err; goto err;
} }
hdesc_size_out = k3_cppi_desc_pool_desc_size(rx_chn->desc_pool);
rx_chn->dsize_log2 = __fls(hdesc_size_out);
WARN_ON(hdesc_size_out != (1 << rx_chn->dsize_log2));
rx_chn->page_pool = NULL;
rx_chn->pages = devm_kcalloc(dev, rx_chn->descs_num,
sizeof(*rx_chn->pages), GFP_KERNEL);
if (!rx_chn->pages)
return -ENOMEM;
common->rx_flow_id_base = common->rx_flow_id_base =
k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn); k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);
dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base); dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base);
...@@ -2252,6 +2714,9 @@ am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx) ...@@ -2252,6 +2714,9 @@ am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
NETIF_F_HW_TC; NETIF_F_HW_TC;
port->ndev->features = port->ndev->hw_features | port->ndev->features = port->ndev->hw_features |
NETIF_F_HW_VLAN_CTAG_FILTER; NETIF_F_HW_VLAN_CTAG_FILTER;
port->ndev->xdp_features = NETDEV_XDP_ACT_BASIC |
NETDEV_XDP_ACT_REDIRECT |
NETDEV_XDP_ACT_NDO_XMIT;
port->ndev->vlan_features |= NETIF_F_SG; port->ndev->vlan_features |= NETIF_F_SG;
port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops; port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops;
port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave; port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave;
...@@ -2315,6 +2780,8 @@ am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx) ...@@ -2315,6 +2780,8 @@ am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
if (ret) if (ret)
dev_err(dev, "failed to add percpu stat free action %d\n", ret); dev_err(dev, "failed to add percpu stat free action %d\n", ret);
port->xdp_prog = NULL;
if (!common->dma_ndev) if (!common->dma_ndev)
common->dma_ndev = port->ndev; common->dma_ndev = port->ndev;
...@@ -2940,9 +3407,9 @@ static int am65_cpsw_nuss_probe(struct platform_device *pdev) ...@@ -2940,9 +3407,9 @@ static int am65_cpsw_nuss_probe(struct platform_device *pdev)
struct device_node *node; struct device_node *node;
struct resource *res; struct resource *res;
struct clk *clk; struct clk *clk;
int ale_entries;
u64 id_temp; u64 id_temp;
int ret, i; int ret, i;
int ale_entries;
common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL); common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL);
if (!common) if (!common)
...@@ -3154,10 +3621,10 @@ static int am65_cpsw_nuss_suspend(struct device *dev) ...@@ -3154,10 +3621,10 @@ static int am65_cpsw_nuss_suspend(struct device *dev)
static int am65_cpsw_nuss_resume(struct device *dev) static int am65_cpsw_nuss_resume(struct device *dev)
{ {
struct am65_cpsw_common *common = dev_get_drvdata(dev); struct am65_cpsw_common *common = dev_get_drvdata(dev);
struct am65_cpsw_host *host_p = am65_common_get_host(common);
struct am65_cpsw_port *port; struct am65_cpsw_port *port;
struct net_device *ndev; struct net_device *ndev;
int i, ret; int i, ret;
struct am65_cpsw_host *host_p = am65_common_get_host(common);
ret = am65_cpsw_nuss_init_tx_chns(common); ret = am65_cpsw_nuss_init_tx_chns(common);
if (ret) if (ret)
......
drivers/net/ethernet/ti/am65-cpsw-nuss.h
View file @ 444cde13
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
#include <linux/platform_device.h> #include <linux/platform_device.h>
#include <linux/soc/ti/k3-ringacc.h> #include <linux/soc/ti/k3-ringacc.h>
#include <net/devlink.h> #include <net/devlink.h>
#include <net/xdp.h>
#include "am65-cpsw-qos.h" #include "am65-cpsw-qos.h"
struct am65_cpts; struct am65_cpts;
...@@ -56,10 +57,18 @@ struct am65_cpsw_port { ...@@ -56,10 +57,18 @@ struct am65_cpsw_port {
bool rx_ts_enabled; bool rx_ts_enabled;
struct am65_cpsw_qos qos; struct am65_cpsw_qos qos;
struct devlink_port devlink_port; struct devlink_port devlink_port;
struct bpf_prog *xdp_prog;
struct xdp_rxq_info xdp_rxq;
/* Only for suspend resume context */ /* Only for suspend resume context */
u32 vid_context; u32 vid_context;
}; };
enum am65_cpsw_tx_buf_type {
AM65_CPSW_TX_BUF_TYPE_SKB,
AM65_CPSW_TX_BUF_TYPE_XDP_TX,
AM65_CPSW_TX_BUF_TYPE_XDP_NDO,
};
struct am65_cpsw_host { struct am65_cpsw_host {
struct am65_cpsw_common *common; struct am65_cpsw_common *common;
void __iomem *port_base; void __iomem *port_base;
...@@ -80,6 +89,7 @@ struct am65_cpsw_tx_chn { ...@@ -80,6 +89,7 @@ struct am65_cpsw_tx_chn {
int irq; int irq;
u32 id; u32 id;
u32 descs_num; u32 descs_num;
unsigned char dsize_log2;
char tx_chn_name[128]; char tx_chn_name[128];
u32 rate_mbps; u32 rate_mbps;
}; };
...@@ -89,7 +99,10 @@ struct am65_cpsw_rx_chn { ...@@ -89,7 +99,10 @@ struct am65_cpsw_rx_chn {
struct device *dma_dev; struct device *dma_dev;
struct k3_cppi_desc_pool *desc_pool; struct k3_cppi_desc_pool *desc_pool;
struct k3_udma_glue_rx_channel *rx_chn; struct k3_udma_glue_rx_channel *rx_chn;
struct page_pool *page_pool;
struct page **pages;
u32 descs_num; u32 descs_num;
unsigned char dsize_log2;
int irq; int irq;
}; };
......
drivers/net/ethernet/ti/k3-cppi-desc-pool.c
View file @ 444cde13
...@@ -22,6 +22,7 @@ struct k3_cppi_desc_pool { ...@@ -22,6 +22,7 @@ struct k3_cppi_desc_pool {
size_t mem_size; size_t mem_size;
size_t num_desc; size_t num_desc;
struct gen_pool *gen_pool; struct gen_pool *gen_pool;
void **desc_infos;
}; };
void k3_cppi_desc_pool_destroy(struct k3_cppi_desc_pool *pool) void k3_cppi_desc_pool_destroy(struct k3_cppi_desc_pool *pool)
...@@ -37,7 +38,11 @@ void k3_cppi_desc_pool_destroy(struct k3_cppi_desc_pool *pool) ...@@ -37,7 +38,11 @@ void k3_cppi_desc_pool_destroy(struct k3_cppi_desc_pool *pool)
dma_free_coherent(pool->dev, pool->mem_size, pool->cpumem, dma_free_coherent(pool->dev, pool->mem_size, pool->cpumem,
pool->dma_addr); pool->dma_addr);
kfree(pool->desc_infos);
gen_pool_destroy(pool->gen_pool); /* frees pool->name */ gen_pool_destroy(pool->gen_pool); /* frees pool->name */
kfree(pool);
} }
EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_destroy); EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_destroy);
...@@ -50,7 +55,7 @@ k3_cppi_desc_pool_create_name(struct device *dev, size_t size, ...@@ -50,7 +55,7 @@ k3_cppi_desc_pool_create_name(struct device *dev, size_t size,
const char *pool_name = NULL; const char *pool_name = NULL;
int ret = -ENOMEM; int ret = -ENOMEM;
pool = devm_kzalloc(dev, sizeof(*pool), GFP_KERNEL); pool = kzalloc(sizeof(*pool), GFP_KERNEL);
if (!pool) if (!pool)
return ERR_PTR(ret); return ERR_PTR(ret);
...@@ -62,18 +67,21 @@ k3_cppi_desc_pool_create_name(struct device *dev, size_t size, ...@@ -62,18 +67,21 @@ k3_cppi_desc_pool_create_name(struct device *dev, size_t size,
pool_name = kstrdup_const(name ? name : dev_name(pool->dev), pool_name = kstrdup_const(name ? name : dev_name(pool->dev),
GFP_KERNEL); GFP_KERNEL);
if (!pool_name) if (!pool_name)
return ERR_PTR(-ENOMEM); goto gen_pool_create_fail;
pool->gen_pool = gen_pool_create(ilog2(pool->desc_size), -1); pool->gen_pool = gen_pool_create(ilog2(pool->desc_size), -1);
if (!pool->gen_pool) { if (!pool->gen_pool) {
ret = -ENOMEM;
dev_err(pool->dev, "pool create failed %d\n", ret);
kfree_const(pool_name); kfree_const(pool_name);
goto gen_pool_create_fail; goto gen_pool_create_fail;
} }
pool->gen_pool->name = pool_name; pool->gen_pool->name = pool_name;
pool->desc_infos = kcalloc(pool->num_desc,
sizeof(*pool->desc_infos), GFP_KERNEL);
if (!pool->desc_infos)
goto gen_pool_desc_infos_alloc_fail;
pool->cpumem = dma_alloc_coherent(pool->dev, pool->mem_size, pool->cpumem = dma_alloc_coherent(pool->dev, pool->mem_size,
&pool->dma_addr, GFP_KERNEL); &pool->dma_addr, GFP_KERNEL);
...@@ -94,9 +102,11 @@ k3_cppi_desc_pool_create_name(struct device *dev, size_t size, ...@@ -94,9 +102,11 @@ k3_cppi_desc_pool_create_name(struct device *dev, size_t size,
dma_free_coherent(pool->dev, pool->mem_size, pool->cpumem, dma_free_coherent(pool->dev, pool->mem_size, pool->cpumem,
pool->dma_addr); pool->dma_addr);
dma_alloc_fail: dma_alloc_fail:
kfree(pool->desc_infos);
gen_pool_desc_infos_alloc_fail:
gen_pool_destroy(pool->gen_pool); /* frees pool->name */ gen_pool_destroy(pool->gen_pool); /* frees pool->name */
gen_pool_create_fail: gen_pool_create_fail:
devm_kfree(pool->dev, pool); kfree(pool);
return ERR_PTR(ret); return ERR_PTR(ret);
} }
EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_create_name); EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_create_name);
...@@ -132,5 +142,31 @@ size_t k3_cppi_desc_pool_avail(struct k3_cppi_desc_pool *pool) ...@@ -132,5 +142,31 @@ size_t k3_cppi_desc_pool_avail(struct k3_cppi_desc_pool *pool)
} }
EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_avail); EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_avail);
size_t k3_cppi_desc_pool_desc_size(const struct k3_cppi_desc_pool *pool)
{
return pool->desc_size;
}
EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_desc_size);
void *k3_cppi_desc_pool_cpuaddr(const struct k3_cppi_desc_pool *pool)
{
return pool->cpumem;
}
EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_cpuaddr);
void k3_cppi_desc_pool_desc_info_set(struct k3_cppi_desc_pool *pool,
int desc_idx, void *info)
{
pool->desc_infos[desc_idx] = info;
}
EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_desc_info_set);
void *k3_cppi_desc_pool_desc_info(const struct k3_cppi_desc_pool *pool,
int desc_idx)
{
return pool->desc_infos[desc_idx];
}
EXPORT_SYMBOL_GPL(k3_cppi_desc_pool_desc_info);
MODULE_LICENSE("GPL"); MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("TI K3 CPPI5 descriptors pool API"); MODULE_DESCRIPTION("TI K3 CPPI5 descriptors pool API");
drivers/net/ethernet/ti/k3-cppi-desc-pool.h
View file @ 444cde13
...@@ -26,5 +26,11 @@ k3_cppi_desc_pool_dma2virt(struct k3_cppi_desc_pool *pool, dma_addr_t dma); ...@@ -26,5 +26,11 @@ k3_cppi_desc_pool_dma2virt(struct k3_cppi_desc_pool *pool, dma_addr_t dma);
void *k3_cppi_desc_pool_alloc(struct k3_cppi_desc_pool *pool); void *k3_cppi_desc_pool_alloc(struct k3_cppi_desc_pool *pool);
void k3_cppi_desc_pool_free(struct k3_cppi_desc_pool *pool, void *addr); void k3_cppi_desc_pool_free(struct k3_cppi_desc_pool *pool, void *addr);
size_t k3_cppi_desc_pool_avail(struct k3_cppi_desc_pool *pool); size_t k3_cppi_desc_pool_avail(struct k3_cppi_desc_pool *pool);
size_t k3_cppi_desc_pool_desc_size(const struct k3_cppi_desc_pool *pool);
void *k3_cppi_desc_pool_cpuaddr(const struct k3_cppi_desc_pool *pool);
void k3_cppi_desc_pool_desc_info_set(struct k3_cppi_desc_pool *pool,
int desc_idx, void *info);
void *k3_cppi_desc_pool_desc_info(const struct k3_cppi_desc_pool *pool,
int desc_idx);
#endif /* K3_CPPI_DESC_POOL_H_ */ #endif /* K3_CPPI_DESC_POOL_H_ */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7