Commit 5413d1ba authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

net: do not block BH while processing socket backlog

Socket backlog processing is a major latency source.

With current TCP socket sk_rcvbuf limits, I have sampled __release_sock()
holding cpu for more than 5 ms, and packets being dropped by the NIC
once ring buffer is filled.

All users are now ready to be called from process context,
we can unblock BH and let interrupts be serviced faster.

cond_resched_softirq() could be removed, as it has no more user.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarSoheil Hassas Yeganeh <soheil@google.com>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 860fbbc3
...@@ -2019,33 +2019,27 @@ static void __release_sock(struct sock *sk) ...@@ -2019,33 +2019,27 @@ static void __release_sock(struct sock *sk)
__releases(&sk->sk_lock.slock) __releases(&sk->sk_lock.slock)
__acquires(&sk->sk_lock.slock) __acquires(&sk->sk_lock.slock)
{ {
struct sk_buff *skb = sk->sk_backlog.head; struct sk_buff *skb, *next;
do { while ((skb = sk->sk_backlog.head) != NULL) {
sk->sk_backlog.head = sk->sk_backlog.tail = NULL; sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
bh_unlock_sock(sk);
do { spin_unlock_bh(&sk->sk_lock.slock);
struct sk_buff *next = skb->next;
do {
next = skb->next;
prefetch(next); prefetch(next);
WARN_ON_ONCE(skb_dst_is_noref(skb)); WARN_ON_ONCE(skb_dst_is_noref(skb));
skb->next = NULL; skb->next = NULL;
sk_backlog_rcv(sk, skb); sk_backlog_rcv(sk, skb);
/* cond_resched();
* We are in process context here with softirqs
* disabled, use cond_resched_softirq() to preempt.
* This is safe to do because we've taken the backlog
* queue private:
*/
cond_resched_softirq();
skb = next; skb = next;
} while (skb != NULL); } while (skb != NULL);
bh_lock_sock(sk); spin_lock_bh(&sk->sk_lock.slock);
} while ((skb = sk->sk_backlog.head) != NULL); }
/* /*
* Doing the zeroing here guarantee we can not loop forever * Doing the zeroing here guarantee we can not loop forever
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment