Commit 2e71a6f8 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

net: gro: selective flush of packets

Current GRO can hold packets in gro_list for almost unlimited
time, in case napi->poll() handler consumes its budget over and over.

In this case, napi_complete()/napi_gro_flush() are not called.

Another problem is that gro_list is flushed in non friendly way :
We scan the list and complete packets in the reverse order.
(youngest packets first, oldest packets last)
This defeats priorities that sender could have cooked.

Since GRO currently only store TCP packets, we dont really notice the
bug because of retransmits, but this behavior can add unexpected
latencies, particularly on mice flows clamped by elephant flows.

This patch makes sure no packet can stay more than 1 ms in queue, and
only in stress situations.

It also complete packets in the right order to minimize latencies.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Jesse Gross <jesse@nicira.com>
Cc: Tom Herbert <therbert@google.com>
Cc: Yuchung Cheng <ycheng@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent a2af139f
...@@ -3189,7 +3189,7 @@ static int skge_poll(struct napi_struct *napi, int to_do) ...@@ -3189,7 +3189,7 @@ static int skge_poll(struct napi_struct *napi, int to_do)
if (work_done < to_do) { if (work_done < to_do) {
unsigned long flags; unsigned long flags;
napi_gro_flush(napi); napi_gro_flush(napi, false);
spin_lock_irqsave(&hw->hw_lock, flags); spin_lock_irqsave(&hw->hw_lock, flags);
__napi_complete(napi); __napi_complete(napi);
hw->intr_mask |= napimask[skge->port]; hw->intr_mask |= napimask[skge->port];
......
...@@ -563,7 +563,7 @@ static int cp_rx_poll(struct napi_struct *napi, int budget) ...@@ -563,7 +563,7 @@ static int cp_rx_poll(struct napi_struct *napi, int budget)
if (cpr16(IntrStatus) & cp_rx_intr_mask) if (cpr16(IntrStatus) & cp_rx_intr_mask)
goto rx_status_loop; goto rx_status_loop;
napi_gro_flush(napi); napi_gro_flush(napi, false);
spin_lock_irqsave(&cp->lock, flags); spin_lock_irqsave(&cp->lock, flags);
__napi_complete(napi); __napi_complete(napi);
cpw16_f(IntrMask, cp_intr_mask); cpw16_f(IntrMask, cp_intr_mask);
......
...@@ -1497,19 +1497,22 @@ struct napi_gro_cb { ...@@ -1497,19 +1497,22 @@ struct napi_gro_cb {
/* This indicates where we are processing relative to skb->data. */ /* This indicates where we are processing relative to skb->data. */
int data_offset; int data_offset;
/* This is non-zero if the packet may be of the same flow. */
int same_flow;
/* This is non-zero if the packet cannot be merged with the new skb. */ /* This is non-zero if the packet cannot be merged with the new skb. */
int flush; int flush;
/* Number of segments aggregated. */ /* Number of segments aggregated. */
int count; u16 count;
/* This is non-zero if the packet may be of the same flow. */
u8 same_flow;
/* Free the skb? */ /* Free the skb? */
int free; u8 free;
#define NAPI_GRO_FREE 1 #define NAPI_GRO_FREE 1
#define NAPI_GRO_FREE_STOLEN_HEAD 2 #define NAPI_GRO_FREE_STOLEN_HEAD 2
/* jiffies when first packet was created/queued */
unsigned long age;
}; };
#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
...@@ -2156,7 +2159,7 @@ extern gro_result_t dev_gro_receive(struct napi_struct *napi, ...@@ -2156,7 +2159,7 @@ extern gro_result_t dev_gro_receive(struct napi_struct *napi,
extern gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb); extern gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb);
extern gro_result_t napi_gro_receive(struct napi_struct *napi, extern gro_result_t napi_gro_receive(struct napi_struct *napi,
struct sk_buff *skb); struct sk_buff *skb);
extern void napi_gro_flush(struct napi_struct *napi); extern void napi_gro_flush(struct napi_struct *napi, bool flush_old);
extern struct sk_buff * napi_get_frags(struct napi_struct *napi); extern struct sk_buff * napi_get_frags(struct napi_struct *napi);
extern gro_result_t napi_frags_finish(struct napi_struct *napi, extern gro_result_t napi_frags_finish(struct napi_struct *napi,
struct sk_buff *skb, struct sk_buff *skb,
......
...@@ -3471,17 +3471,31 @@ static int napi_gro_complete(struct sk_buff *skb) ...@@ -3471,17 +3471,31 @@ static int napi_gro_complete(struct sk_buff *skb)
return netif_receive_skb(skb); return netif_receive_skb(skb);
} }
inline void napi_gro_flush(struct napi_struct *napi) /* napi->gro_list contains packets ordered by age.
* youngest packets at the head of it.
* Complete skbs in reverse order to reduce latencies.
*/
void napi_gro_flush(struct napi_struct *napi, bool flush_old)
{ {
struct sk_buff *skb, *next; struct sk_buff *skb, *prev = NULL;
for (skb = napi->gro_list; skb; skb = next) { /* scan list and build reverse chain */
next = skb->next; for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
skb->prev = prev;
prev = skb;
}
for (skb = prev; skb; skb = prev) {
skb->next = NULL; skb->next = NULL;
if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
return;
prev = skb->prev;
napi_gro_complete(skb); napi_gro_complete(skb);
napi->gro_count--;
} }
napi->gro_count = 0;
napi->gro_list = NULL; napi->gro_list = NULL;
} }
EXPORT_SYMBOL(napi_gro_flush); EXPORT_SYMBOL(napi_gro_flush);
...@@ -3542,6 +3556,7 @@ enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) ...@@ -3542,6 +3556,7 @@ enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
napi->gro_count++; napi->gro_count++;
NAPI_GRO_CB(skb)->count = 1; NAPI_GRO_CB(skb)->count = 1;
NAPI_GRO_CB(skb)->age = jiffies;
skb_shinfo(skb)->gso_size = skb_gro_len(skb); skb_shinfo(skb)->gso_size = skb_gro_len(skb);
skb->next = napi->gro_list; skb->next = napi->gro_list;
napi->gro_list = skb; napi->gro_list = skb;
...@@ -3878,7 +3893,7 @@ void napi_complete(struct napi_struct *n) ...@@ -3878,7 +3893,7 @@ void napi_complete(struct napi_struct *n)
if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state))) if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
return; return;
napi_gro_flush(n); napi_gro_flush(n, false);
local_irq_save(flags); local_irq_save(flags);
__napi_complete(n); __napi_complete(n);
local_irq_restore(flags); local_irq_restore(flags);
...@@ -3983,8 +3998,17 @@ static void net_rx_action(struct softirq_action *h) ...@@ -3983,8 +3998,17 @@ static void net_rx_action(struct softirq_action *h)
local_irq_enable(); local_irq_enable();
napi_complete(n); napi_complete(n);
local_irq_disable(); local_irq_disable();
} else } else {
if (n->gro_list) {
/* flush too old packets
* If HZ < 1000, flush all packets.
*/
local_irq_enable();
napi_gro_flush(n, HZ >= 1000);
local_irq_disable();
}
list_move_tail(&n->poll_list, &sd->poll_list); list_move_tail(&n->poll_list, &sd->poll_list);
}
} }
netpoll_poll_unlock(have); netpoll_poll_unlock(have);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment