Commit 05e22e83 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

tcp: remove indirect calls for icsk->icsk_af_ops->queue_xmit

Mitigate RETPOLINE costs in __tcp_transmit_skb()
by using INDIRECT_CALL_INET() wrapper.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 902053f1
...@@ -231,11 +231,7 @@ struct sk_buff *ip_make_skb(struct sock *sk, struct flowi4 *fl4, ...@@ -231,11 +231,7 @@ struct sk_buff *ip_make_skb(struct sock *sk, struct flowi4 *fl4,
struct ipcm_cookie *ipc, struct rtable **rtp, struct ipcm_cookie *ipc, struct rtable **rtp,
struct inet_cork *cork, unsigned int flags); struct inet_cork *cork, unsigned int flags);
static inline int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl);
struct flowi *fl)
{
return __ip_queue_xmit(sk, skb, fl, inet_sk(sk)->tos);
}
static inline struct sk_buff *ip_finish_skb(struct sock *sk, struct flowi4 *fl4) static inline struct sk_buff *ip_finish_skb(struct sock *sk, struct flowi4 *fl4)
{ {
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
#include <linux/skbuff.h> #include <linux/skbuff.h>
#include <linux/kref.h> #include <linux/kref.h>
#include <linux/ktime.h> #include <linux/ktime.h>
#include <linux/indirect_call_wrapper.h>
#include <net/inet_connection_sock.h> #include <net/inet_connection_sock.h>
#include <net/inet_timewait_sock.h> #include <net/inet_timewait_sock.h>
......
...@@ -539,6 +539,12 @@ int __ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl, ...@@ -539,6 +539,12 @@ int __ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl,
} }
EXPORT_SYMBOL(__ip_queue_xmit); EXPORT_SYMBOL(__ip_queue_xmit);
int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)
{
return __ip_queue_xmit(sk, skb, fl, inet_sk(sk)->tos);
}
EXPORT_SYMBOL(ip_queue_xmit);
static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from) static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
{ {
to->pkt_type = from->pkt_type; to->pkt_type = from->pkt_type;
......
...@@ -1064,6 +1064,9 @@ static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb, ...@@ -1064,6 +1064,9 @@ static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb,
list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
} }
INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
/* This routine actually transmits TCP packets queued in by /* This routine actually transmits TCP packets queued in by
* tcp_do_sendmsg(). This is used by both the initial * tcp_do_sendmsg(). This is used by both the initial
* transmission and possible later retransmissions. * transmission and possible later retransmissions.
...@@ -1235,7 +1238,9 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, ...@@ -1235,7 +1238,9 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
tcp_add_tx_delay(skb, tp); tcp_add_tx_delay(skb, tp);
err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl); err = INDIRECT_CALL_INET(icsk->icsk_af_ops->queue_xmit,
inet6_csk_xmit, ip_queue_xmit,
sk, skb, &inet->cork.fl);
if (unlikely(err > 0)) { if (unlikely(err > 0)) {
tcp_enter_cwr(sk); tcp_enter_cwr(sk);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment