Commit 1abe267f authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski

net: add sk_wake_async_rcu() helper

While looking at UDP receive performance, I saw sk_wake_async()
was no longer inlined.

This matters at least on AMD Zen1-4 platforms (see SRSO)

This might be because rcu_read_lock() and rcu_read_unlock()
are no longer nops in recent kernels ?

Add sk_wake_async_rcu() variant, which must be called from
contexts already holding rcu lock.

As SOCK_FASYNC is deprecated in modern days, use unlikely()
to give a hint to the compiler.

sk_wake_async_rcu() is properly inlined from
__udp_enqueue_schedule_skb() and sock_def_readable().
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Link: https://lore.kernel.org/r/20240328144032.1864988-5-edumazet@google.comSigned-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 612b1c0d
...@@ -847,7 +847,7 @@ void af_alg_wmem_wakeup(struct sock *sk) ...@@ -847,7 +847,7 @@ void af_alg_wmem_wakeup(struct sock *sk)
wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
EPOLLRDNORM | EPOLLRDNORM |
EPOLLRDBAND); EPOLLRDBAND);
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
rcu_read_unlock(); rcu_read_unlock();
} }
EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup); EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup);
...@@ -914,7 +914,7 @@ static void af_alg_data_wakeup(struct sock *sk) ...@@ -914,7 +914,7 @@ static void af_alg_data_wakeup(struct sock *sk)
wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
EPOLLRDNORM | EPOLLRDNORM |
EPOLLRDBAND); EPOLLRDBAND);
sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
rcu_read_unlock(); rcu_read_unlock();
} }
......
...@@ -2513,6 +2513,12 @@ static inline void sk_wake_async(const struct sock *sk, int how, int band) ...@@ -2513,6 +2513,12 @@ static inline void sk_wake_async(const struct sock *sk, int how, int band)
} }
} }
static inline void sk_wake_async_rcu(const struct sock *sk, int how, int band)
{
if (unlikely(sock_flag(sk, SOCK_FASYNC)))
sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
}
/* Since sk_{r,w}mem_alloc sums skb->truesize, even a small frame might /* Since sk_{r,w}mem_alloc sums skb->truesize, even a small frame might
* need sizeof(sk_buff) + MTU + padding, unless net driver perform copybreak. * need sizeof(sk_buff) + MTU + padding, unless net driver perform copybreak.
* Note: for send buffers, TCP works better if we can build two skbs at * Note: for send buffers, TCP works better if we can build two skbs at
......
...@@ -116,7 +116,7 @@ static void vcc_write_space(struct sock *sk) ...@@ -116,7 +116,7 @@ static void vcc_write_space(struct sock *sk)
if (skwq_has_sleeper(wq)) if (skwq_has_sleeper(wq))
wake_up_interruptible(&wq->wait); wake_up_interruptible(&wq->wait);
sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
} }
rcu_read_unlock(); rcu_read_unlock();
......
...@@ -3338,7 +3338,7 @@ static void sock_def_error_report(struct sock *sk) ...@@ -3338,7 +3338,7 @@ static void sock_def_error_report(struct sock *sk)
wq = rcu_dereference(sk->sk_wq); wq = rcu_dereference(sk->sk_wq);
if (skwq_has_sleeper(wq)) if (skwq_has_sleeper(wq))
wake_up_interruptible_poll(&wq->wait, EPOLLERR); wake_up_interruptible_poll(&wq->wait, EPOLLERR);
sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); sk_wake_async_rcu(sk, SOCK_WAKE_IO, POLL_ERR);
rcu_read_unlock(); rcu_read_unlock();
} }
...@@ -3353,7 +3353,7 @@ void sock_def_readable(struct sock *sk) ...@@ -3353,7 +3353,7 @@ void sock_def_readable(struct sock *sk)
if (skwq_has_sleeper(wq)) if (skwq_has_sleeper(wq))
wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
EPOLLRDNORM | EPOLLRDBAND); EPOLLRDNORM | EPOLLRDBAND);
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
rcu_read_unlock(); rcu_read_unlock();
} }
...@@ -3373,7 +3373,7 @@ static void sock_def_write_space(struct sock *sk) ...@@ -3373,7 +3373,7 @@ static void sock_def_write_space(struct sock *sk)
EPOLLWRNORM | EPOLLWRBAND); EPOLLWRNORM | EPOLLWRBAND);
/* Should agree with poll, otherwise some programs break */ /* Should agree with poll, otherwise some programs break */
sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
} }
rcu_read_unlock(); rcu_read_unlock();
...@@ -3398,7 +3398,7 @@ static void sock_def_write_space_wfree(struct sock *sk) ...@@ -3398,7 +3398,7 @@ static void sock_def_write_space_wfree(struct sock *sk)
EPOLLWRNORM | EPOLLWRBAND); EPOLLWRNORM | EPOLLWRBAND);
/* Should agree with poll, otherwise some programs break */ /* Should agree with poll, otherwise some programs break */
sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
} }
} }
......
...@@ -204,7 +204,7 @@ void dccp_write_space(struct sock *sk) ...@@ -204,7 +204,7 @@ void dccp_write_space(struct sock *sk)
wake_up_interruptible(&wq->wait); wake_up_interruptible(&wq->wait);
/* Should agree with poll, otherwise some programs break */ /* Should agree with poll, otherwise some programs break */
if (sock_writeable(sk)) if (sock_writeable(sk))
sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
rcu_read_unlock(); rcu_read_unlock();
} }
......
...@@ -1544,7 +1544,7 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb) ...@@ -1544,7 +1544,7 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
INDIRECT_CALL_1(sk->sk_data_ready, INDIRECT_CALL_1(sk->sk_data_ready,
sock_def_readable, sk); sock_def_readable, sk);
else else
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
} }
busylock_release(busy); busylock_release(busy);
return 0; return 0;
......
...@@ -184,7 +184,7 @@ static void iucv_sock_wake_msglim(struct sock *sk) ...@@ -184,7 +184,7 @@ static void iucv_sock_wake_msglim(struct sock *sk)
wq = rcu_dereference(sk->sk_wq); wq = rcu_dereference(sk->sk_wq);
if (skwq_has_sleeper(wq)) if (skwq_has_sleeper(wq))
wake_up_interruptible_all(&wq->wait); wake_up_interruptible_all(&wq->wait);
sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
rcu_read_unlock(); rcu_read_unlock();
} }
......
...@@ -65,7 +65,7 @@ static void rxrpc_write_space(struct sock *sk) ...@@ -65,7 +65,7 @@ static void rxrpc_write_space(struct sock *sk)
if (skwq_has_sleeper(wq)) if (skwq_has_sleeper(wq))
wake_up_interruptible(&wq->wait); wake_up_interruptible(&wq->wait);
sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
} }
rcu_read_unlock(); rcu_read_unlock();
} }
......
...@@ -9276,7 +9276,7 @@ void sctp_data_ready(struct sock *sk) ...@@ -9276,7 +9276,7 @@ void sctp_data_ready(struct sock *sk)
if (skwq_has_sleeper(wq)) if (skwq_has_sleeper(wq))
wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
EPOLLRDNORM | EPOLLRDBAND); EPOLLRDNORM | EPOLLRDBAND);
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
rcu_read_unlock(); rcu_read_unlock();
} }
......
...@@ -42,10 +42,10 @@ static void smc_rx_wake_up(struct sock *sk) ...@@ -42,10 +42,10 @@ static void smc_rx_wake_up(struct sock *sk)
if (skwq_has_sleeper(wq)) if (skwq_has_sleeper(wq))
wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
EPOLLRDNORM | EPOLLRDBAND); EPOLLRDNORM | EPOLLRDBAND);
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
if ((sk->sk_shutdown == SHUTDOWN_MASK) || if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
(sk->sk_state == SMC_CLOSED)) (sk->sk_state == SMC_CLOSED))
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_HUP);
rcu_read_unlock(); rcu_read_unlock();
} }
......
...@@ -546,7 +546,7 @@ static void unix_write_space(struct sock *sk) ...@@ -546,7 +546,7 @@ static void unix_write_space(struct sock *sk)
if (skwq_has_sleeper(wq)) if (skwq_has_sleeper(wq))
wake_up_interruptible_sync_poll(&wq->wait, wake_up_interruptible_sync_poll(&wq->wait,
EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND); EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
} }
rcu_read_unlock(); rcu_read_unlock();
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment