Commit 4055eaef authored by Petri Gynther's avatar Petri Gynther Committed by David S. Miller

net: bcmgenet: add support for multiple Rx queues

Add support for multiple Rx queues:
1. Add NAPI context per Rx queue
2. Modify Rx interrupt and Rx NAPI code to handle multiple Rx queues
Signed-off-by: default avatarPetri Gynther <pgynther@google.com>
Reviewed-by: default avatarFlorian Fainelli <f.fainelli@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 3ab11339
...@@ -964,6 +964,34 @@ static void bcmgenet_free_cb(struct enet_cb *cb) ...@@ -964,6 +964,34 @@ static void bcmgenet_free_cb(struct enet_cb *cb)
dma_unmap_addr_set(cb, dma_addr, 0); dma_unmap_addr_set(cb, dma_addr, 0);
} }
static inline void bcmgenet_rx_ring16_int_disable(struct bcmgenet_rx_ring *ring)
{
bcmgenet_intrl2_0_writel(ring->priv,
UMAC_IRQ_RXDMA_BDONE | UMAC_IRQ_RXDMA_PDONE,
INTRL2_CPU_MASK_SET);
}
static inline void bcmgenet_rx_ring16_int_enable(struct bcmgenet_rx_ring *ring)
{
bcmgenet_intrl2_0_writel(ring->priv,
UMAC_IRQ_RXDMA_BDONE | UMAC_IRQ_RXDMA_PDONE,
INTRL2_CPU_MASK_CLEAR);
}
static inline void bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring *ring)
{
bcmgenet_intrl2_1_writel(ring->priv,
1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
INTRL2_CPU_MASK_SET);
}
static inline void bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring *ring)
{
bcmgenet_intrl2_1_writel(ring->priv,
1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
INTRL2_CPU_MASK_CLEAR);
}
static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_tx_ring *ring) static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_tx_ring *ring)
{ {
bcmgenet_intrl2_0_writel(ring->priv, bcmgenet_intrl2_0_writel(ring->priv,
...@@ -1390,11 +1418,10 @@ static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv, ...@@ -1390,11 +1418,10 @@ static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv,
/* bcmgenet_desc_rx - descriptor based rx process. /* bcmgenet_desc_rx - descriptor based rx process.
* this could be called from bottom half, or from NAPI polling method. * this could be called from bottom half, or from NAPI polling method.
*/ */
static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv, static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
unsigned int index,
unsigned int budget) unsigned int budget)
{ {
struct bcmgenet_rx_ring *ring = &priv->rx_rings[index]; struct bcmgenet_priv *priv = ring->priv;
struct net_device *dev = priv->dev; struct net_device *dev = priv->dev;
struct enet_cb *cb; struct enet_cb *cb;
struct sk_buff *skb; struct sk_buff *skb;
...@@ -1406,7 +1433,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv, ...@@ -1406,7 +1433,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv,
unsigned int discards; unsigned int discards;
unsigned int chksum_ok = 0; unsigned int chksum_ok = 0;
p_index = bcmgenet_rdma_ring_readl(priv, index, RDMA_PROD_INDEX); p_index = bcmgenet_rdma_ring_readl(priv, ring->index, RDMA_PROD_INDEX);
discards = (p_index >> DMA_P_INDEX_DISCARD_CNT_SHIFT) & discards = (p_index >> DMA_P_INDEX_DISCARD_CNT_SHIFT) &
DMA_P_INDEX_DISCARD_CNT_MASK; DMA_P_INDEX_DISCARD_CNT_MASK;
...@@ -1419,7 +1446,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv, ...@@ -1419,7 +1446,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv,
/* Clear HW register when we reach 75% of maximum 0xFFFF */ /* Clear HW register when we reach 75% of maximum 0xFFFF */
if (ring->old_discards >= 0xC000) { if (ring->old_discards >= 0xC000) {
ring->old_discards = 0; ring->old_discards = 0;
bcmgenet_rdma_ring_writel(priv, index, 0, bcmgenet_rdma_ring_writel(priv, ring->index, 0,
RDMA_PROD_INDEX); RDMA_PROD_INDEX);
} }
} }
...@@ -1527,7 +1554,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv, ...@@ -1527,7 +1554,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv,
dev->stats.multicast++; dev->stats.multicast++;
/* Notify kernel */ /* Notify kernel */
napi_gro_receive(&priv->napi, skb); napi_gro_receive(&ring->napi, skb);
netif_dbg(priv, rx_status, dev, "pushed up to kernel\n"); netif_dbg(priv, rx_status, dev, "pushed up to kernel\n");
next: next:
...@@ -1538,7 +1565,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv, ...@@ -1538,7 +1565,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv,
ring->read_ptr = ring->cb_ptr; ring->read_ptr = ring->cb_ptr;
ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK; ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK;
bcmgenet_rdma_ring_writel(priv, index, ring->c_index, RDMA_CONS_INDEX); bcmgenet_rdma_ring_writel(priv, ring->index, ring->c_index, RDMA_CONS_INDEX);
} }
return rxpktprocessed; return rxpktprocessed;
...@@ -1547,17 +1574,15 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv, ...@@ -1547,17 +1574,15 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv,
/* Rx NAPI polling method */ /* Rx NAPI polling method */
static int bcmgenet_rx_poll(struct napi_struct *napi, int budget) static int bcmgenet_rx_poll(struct napi_struct *napi, int budget)
{ {
struct bcmgenet_priv *priv = container_of(napi, struct bcmgenet_rx_ring *ring = container_of(napi,
struct bcmgenet_priv, napi); struct bcmgenet_rx_ring, napi);
unsigned int work_done; unsigned int work_done;
work_done = bcmgenet_desc_rx(priv, DESC_INDEX, budget); work_done = bcmgenet_desc_rx(ring, budget);
if (work_done < budget) { if (work_done < budget) {
napi_complete(napi); napi_complete(napi);
bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_RXDMA_BDONE | ring->int_enable(ring);
UMAC_IRQ_RXDMA_PDONE,
INTRL2_CPU_MASK_CLEAR);
} }
return work_done; return work_done;
...@@ -1728,6 +1753,10 @@ static int init_umac(struct bcmgenet_priv *priv) ...@@ -1728,6 +1753,10 @@ static int init_umac(struct bcmgenet_priv *priv)
if (priv->hw_params->flags & GENET_HAS_MDIO_INTR) if (priv->hw_params->flags & GENET_HAS_MDIO_INTR)
int0_enable |= (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR); int0_enable |= (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR);
/* Enable Rx priority queue interrupts */
for (i = 0; i < priv->hw_params->rx_queues; ++i)
int1_enable |= (1 << (UMAC_IRQ1_RX_INTR_SHIFT + i));
/* Enable Tx priority queue interrupts */ /* Enable Tx priority queue interrupts */
for (i = 0; i < priv->hw_params->tx_queues; ++i) for (i = 0; i < priv->hw_params->tx_queues; ++i)
int1_enable |= (1 << i); int1_enable |= (1 << i);
...@@ -1806,7 +1835,15 @@ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv, ...@@ -1806,7 +1835,15 @@ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv,
u32 words_per_bd = WORDS_PER_BD(priv); u32 words_per_bd = WORDS_PER_BD(priv);
int ret; int ret;
ring->priv = priv;
ring->index = index; ring->index = index;
if (index == DESC_INDEX) {
ring->int_enable = bcmgenet_rx_ring16_int_enable;
ring->int_disable = bcmgenet_rx_ring16_int_disable;
} else {
ring->int_enable = bcmgenet_rx_ring_int_enable;
ring->int_disable = bcmgenet_rx_ring_int_disable;
}
ring->cbs = priv->rx_cbs + start_ptr; ring->cbs = priv->rx_cbs + start_ptr;
ring->size = size; ring->size = size;
ring->c_index = 0; ring->c_index = 0;
...@@ -1972,22 +2009,58 @@ static void bcmgenet_init_tx_queues(struct net_device *dev) ...@@ -1972,22 +2009,58 @@ static void bcmgenet_init_tx_queues(struct net_device *dev)
static void bcmgenet_init_rx_napi(struct bcmgenet_priv *priv) static void bcmgenet_init_rx_napi(struct bcmgenet_priv *priv)
{ {
netif_napi_add(priv->dev, &priv->napi, bcmgenet_rx_poll, 64); unsigned int i;
struct bcmgenet_rx_ring *ring;
for (i = 0; i < priv->hw_params->rx_queues; ++i) {
ring = &priv->rx_rings[i];
netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll, 64);
}
ring = &priv->rx_rings[DESC_INDEX];
netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll, 64);
} }
static void bcmgenet_enable_rx_napi(struct bcmgenet_priv *priv) static void bcmgenet_enable_rx_napi(struct bcmgenet_priv *priv)
{ {
napi_enable(&priv->napi); unsigned int i;
struct bcmgenet_rx_ring *ring;
for (i = 0; i < priv->hw_params->rx_queues; ++i) {
ring = &priv->rx_rings[i];
napi_enable(&ring->napi);
}
ring = &priv->rx_rings[DESC_INDEX];
napi_enable(&ring->napi);
} }
static void bcmgenet_disable_rx_napi(struct bcmgenet_priv *priv) static void bcmgenet_disable_rx_napi(struct bcmgenet_priv *priv)
{ {
napi_disable(&priv->napi); unsigned int i;
struct bcmgenet_rx_ring *ring;
for (i = 0; i < priv->hw_params->rx_queues; ++i) {
ring = &priv->rx_rings[i];
napi_disable(&ring->napi);
}
ring = &priv->rx_rings[DESC_INDEX];
napi_disable(&ring->napi);
} }
static void bcmgenet_fini_rx_napi(struct bcmgenet_priv *priv) static void bcmgenet_fini_rx_napi(struct bcmgenet_priv *priv)
{ {
netif_napi_del(&priv->napi); unsigned int i;
struct bcmgenet_rx_ring *ring;
for (i = 0; i < priv->hw_params->rx_queues; ++i) {
ring = &priv->rx_rings[i];
netif_napi_del(&ring->napi);
}
ring = &priv->rx_rings[DESC_INDEX];
netif_napi_del(&ring->napi);
} }
/* Initialize Rx queues /* Initialize Rx queues
...@@ -2214,50 +2287,66 @@ static void bcmgenet_irq_task(struct work_struct *work) ...@@ -2214,50 +2287,66 @@ static void bcmgenet_irq_task(struct work_struct *work)
} }
} }
/* bcmgenet_isr1: interrupt handler for ring buffer. */ /* bcmgenet_isr1: handle Rx and Tx priority queues */
static irqreturn_t bcmgenet_isr1(int irq, void *dev_id) static irqreturn_t bcmgenet_isr1(int irq, void *dev_id)
{ {
struct bcmgenet_priv *priv = dev_id; struct bcmgenet_priv *priv = dev_id;
struct bcmgenet_tx_ring *ring; struct bcmgenet_rx_ring *rx_ring;
struct bcmgenet_tx_ring *tx_ring;
unsigned int index; unsigned int index;
/* Save irq status for bottom-half processing. */ /* Save irq status for bottom-half processing. */
priv->irq1_stat = priv->irq1_stat =
bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_STAT) & bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_STAT) &
~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS); ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
/* clear interrupts */ /* clear interrupts */
bcmgenet_intrl2_1_writel(priv, priv->irq1_stat, INTRL2_CPU_CLEAR); bcmgenet_intrl2_1_writel(priv, priv->irq1_stat, INTRL2_CPU_CLEAR);
netif_dbg(priv, intr, priv->dev, netif_dbg(priv, intr, priv->dev,
"%s: IRQ=0x%x\n", __func__, priv->irq1_stat); "%s: IRQ=0x%x\n", __func__, priv->irq1_stat);
/* Check the MBDONE interrupts. /* Check Rx priority queue interrupts */
* packet is done, reclaim descriptors for (index = 0; index < priv->hw_params->rx_queues; index++) {
*/ if (!(priv->irq1_stat & BIT(UMAC_IRQ1_RX_INTR_SHIFT + index)))
continue;
rx_ring = &priv->rx_rings[index];
if (likely(napi_schedule_prep(&rx_ring->napi))) {
rx_ring->int_disable(rx_ring);
__napi_schedule(&rx_ring->napi);
}
}
/* Check Tx priority queue interrupts */
for (index = 0; index < priv->hw_params->tx_queues; index++) { for (index = 0; index < priv->hw_params->tx_queues; index++) {
if (!(priv->irq1_stat & BIT(index))) if (!(priv->irq1_stat & BIT(index)))
continue; continue;
ring = &priv->tx_rings[index]; tx_ring = &priv->tx_rings[index];
if (likely(napi_schedule_prep(&ring->napi))) { if (likely(napi_schedule_prep(&tx_ring->napi))) {
ring->int_disable(ring); tx_ring->int_disable(tx_ring);
__napi_schedule(&ring->napi); __napi_schedule(&tx_ring->napi);
} }
} }
return IRQ_HANDLED; return IRQ_HANDLED;
} }
/* bcmgenet_isr0: Handle various interrupts. */ /* bcmgenet_isr0: handle Rx and Tx default queues + other stuff */
static irqreturn_t bcmgenet_isr0(int irq, void *dev_id) static irqreturn_t bcmgenet_isr0(int irq, void *dev_id)
{ {
struct bcmgenet_priv *priv = dev_id; struct bcmgenet_priv *priv = dev_id;
struct bcmgenet_rx_ring *rx_ring;
struct bcmgenet_tx_ring *tx_ring;
/* Save irq status for bottom-half processing. */ /* Save irq status for bottom-half processing. */
priv->irq0_stat = priv->irq0_stat =
bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT) & bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT) &
~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS); ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
/* clear interrupts */ /* clear interrupts */
bcmgenet_intrl2_0_writel(priv, priv->irq0_stat, INTRL2_CPU_CLEAR); bcmgenet_intrl2_0_writel(priv, priv->irq0_stat, INTRL2_CPU_CLEAR);
...@@ -2265,26 +2354,23 @@ static irqreturn_t bcmgenet_isr0(int irq, void *dev_id) ...@@ -2265,26 +2354,23 @@ static irqreturn_t bcmgenet_isr0(int irq, void *dev_id)
"IRQ=0x%x\n", priv->irq0_stat); "IRQ=0x%x\n", priv->irq0_stat);
if (priv->irq0_stat & (UMAC_IRQ_RXDMA_BDONE | UMAC_IRQ_RXDMA_PDONE)) { if (priv->irq0_stat & (UMAC_IRQ_RXDMA_BDONE | UMAC_IRQ_RXDMA_PDONE)) {
/* We use NAPI(software interrupt throttling, if rx_ring = &priv->rx_rings[DESC_INDEX];
* Rx Descriptor throttling is not used.
* Disable interrupt, will be enabled in the poll method. if (likely(napi_schedule_prep(&rx_ring->napi))) {
*/ rx_ring->int_disable(rx_ring);
if (likely(napi_schedule_prep(&priv->napi))) { __napi_schedule(&rx_ring->napi);
bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_RXDMA_BDONE |
UMAC_IRQ_RXDMA_PDONE,
INTRL2_CPU_MASK_SET);
__napi_schedule(&priv->napi);
} }
} }
if (priv->irq0_stat &
(UMAC_IRQ_TXDMA_BDONE | UMAC_IRQ_TXDMA_PDONE)) {
struct bcmgenet_tx_ring *ring = &priv->tx_rings[DESC_INDEX];
if (likely(napi_schedule_prep(&ring->napi))) { if (priv->irq0_stat & (UMAC_IRQ_TXDMA_BDONE | UMAC_IRQ_TXDMA_PDONE)) {
ring->int_disable(ring); tx_ring = &priv->tx_rings[DESC_INDEX];
__napi_schedule(&ring->napi);
if (likely(napi_schedule_prep(&tx_ring->napi))) {
tx_ring->int_disable(tx_ring);
__napi_schedule(&tx_ring->napi);
} }
} }
if (priv->irq0_stat & (UMAC_IRQ_PHY_DET_R | if (priv->irq0_stat & (UMAC_IRQ_PHY_DET_R |
UMAC_IRQ_PHY_DET_F | UMAC_IRQ_PHY_DET_F |
UMAC_IRQ_LINK_UP | UMAC_IRQ_LINK_UP |
......
...@@ -310,6 +310,11 @@ struct bcmgenet_mib_counters { ...@@ -310,6 +310,11 @@ struct bcmgenet_mib_counters {
#define UMAC_IRQ_MDIO_DONE (1 << 23) #define UMAC_IRQ_MDIO_DONE (1 << 23)
#define UMAC_IRQ_MDIO_ERROR (1 << 24) #define UMAC_IRQ_MDIO_ERROR (1 << 24)
/* INTRL2 instance 1 definitions */
#define UMAC_IRQ1_TX_INTR_MASK 0xFFFF
#define UMAC_IRQ1_RX_INTR_MASK 0xFFFF
#define UMAC_IRQ1_RX_INTR_SHIFT 16
/* Register block offsets */ /* Register block offsets */
#define GENET_SYS_OFF 0x0000 #define GENET_SYS_OFF 0x0000
#define GENET_GR_BRIDGE_OFF 0x0040 #define GENET_GR_BRIDGE_OFF 0x0040
...@@ -541,6 +546,7 @@ struct bcmgenet_tx_ring { ...@@ -541,6 +546,7 @@ struct bcmgenet_tx_ring {
}; };
struct bcmgenet_rx_ring { struct bcmgenet_rx_ring {
struct napi_struct napi; /* Rx NAPI struct */
unsigned int index; /* Rx ring index */ unsigned int index; /* Rx ring index */
struct enet_cb *cbs; /* Rx ring buffer control block */ struct enet_cb *cbs; /* Rx ring buffer control block */
unsigned int size; /* Rx ring size */ unsigned int size; /* Rx ring size */
...@@ -549,6 +555,9 @@ struct bcmgenet_rx_ring { ...@@ -549,6 +555,9 @@ struct bcmgenet_rx_ring {
unsigned int cb_ptr; /* Rx ring initial CB ptr */ unsigned int cb_ptr; /* Rx ring initial CB ptr */
unsigned int end_ptr; /* Rx ring end CB ptr */ unsigned int end_ptr; /* Rx ring end CB ptr */
unsigned int old_discards; unsigned int old_discards;
void (*int_enable)(struct bcmgenet_rx_ring *);
void (*int_disable)(struct bcmgenet_rx_ring *);
struct bcmgenet_priv *priv;
}; };
/* device context */ /* device context */
...@@ -557,9 +566,6 @@ struct bcmgenet_priv { ...@@ -557,9 +566,6 @@ struct bcmgenet_priv {
enum bcmgenet_version version; enum bcmgenet_version version;
struct net_device *dev; struct net_device *dev;
/* NAPI for descriptor based rx */
struct napi_struct napi ____cacheline_aligned;
/* transmit variables */ /* transmit variables */
void __iomem *tx_bds; void __iomem *tx_bds;
struct enet_cb *tx_cbs; struct enet_cb *tx_cbs;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment