Commit 1485348d authored by Ben Hutchings's avatar Ben Hutchings Committed by David S. Miller

tcp: Apply device TSO segment limit earlier

Cache the device gso_max_segs in sock::sk_gso_max_segs and use it to
limit the size of TSO skbs.  This avoids the need to fall back to
software GSO for local TCP senders.
Signed-off-by: default avatarBen Hutchings <bhutchings@solarflare.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 7e6d06f0
...@@ -218,6 +218,7 @@ struct cg_proto; ...@@ -218,6 +218,7 @@ struct cg_proto;
* @sk_route_nocaps: forbidden route capabilities (e.g NETIF_F_GSO_MASK) * @sk_route_nocaps: forbidden route capabilities (e.g NETIF_F_GSO_MASK)
* @sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4) * @sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)
* @sk_gso_max_size: Maximum GSO segment size to build * @sk_gso_max_size: Maximum GSO segment size to build
* @sk_gso_max_segs: Maximum number of GSO segments
* @sk_lingertime: %SO_LINGER l_linger setting * @sk_lingertime: %SO_LINGER l_linger setting
* @sk_backlog: always used with the per-socket spinlock held * @sk_backlog: always used with the per-socket spinlock held
* @sk_callback_lock: used with the callbacks in the end of this struct * @sk_callback_lock: used with the callbacks in the end of this struct
...@@ -338,6 +339,7 @@ struct sock { ...@@ -338,6 +339,7 @@ struct sock {
netdev_features_t sk_route_nocaps; netdev_features_t sk_route_nocaps;
int sk_gso_type; int sk_gso_type;
unsigned int sk_gso_max_size; unsigned int sk_gso_max_size;
u16 sk_gso_max_segs;
int sk_rcvlowat; int sk_rcvlowat;
unsigned long sk_lingertime; unsigned long sk_lingertime;
struct sk_buff_head sk_error_queue; struct sk_buff_head sk_error_queue;
......
...@@ -1458,6 +1458,7 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst) ...@@ -1458,6 +1458,7 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
} else { } else {
sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
sk->sk_gso_max_size = dst->dev->gso_max_size; sk->sk_gso_max_size = dst->dev->gso_max_size;
sk->sk_gso_max_segs = dst->dev->gso_max_segs;
} }
} }
} }
......
...@@ -811,7 +811,9 @@ static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now, ...@@ -811,7 +811,9 @@ static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
old_size_goal + mss_now > xmit_size_goal)) { old_size_goal + mss_now > xmit_size_goal)) {
xmit_size_goal = old_size_goal; xmit_size_goal = old_size_goal;
} else { } else {
tp->xmit_size_goal_segs = xmit_size_goal / mss_now; tp->xmit_size_goal_segs =
min_t(u16, xmit_size_goal / mss_now,
sk->sk_gso_max_segs);
xmit_size_goal = tp->xmit_size_goal_segs * mss_now; xmit_size_goal = tp->xmit_size_goal_segs * mss_now;
} }
} }
......
...@@ -291,7 +291,8 @@ bool tcp_is_cwnd_limited(const struct sock *sk, u32 in_flight) ...@@ -291,7 +291,8 @@ bool tcp_is_cwnd_limited(const struct sock *sk, u32 in_flight)
left = tp->snd_cwnd - in_flight; left = tp->snd_cwnd - in_flight;
if (sk_can_gso(sk) && if (sk_can_gso(sk) &&
left * sysctl_tcp_tso_win_divisor < tp->snd_cwnd && left * sysctl_tcp_tso_win_divisor < tp->snd_cwnd &&
left * tp->mss_cache < sk->sk_gso_max_size) left * tp->mss_cache < sk->sk_gso_max_size &&
left < sk->sk_gso_max_segs)
return true; return true;
return left <= tcp_max_tso_deferred_mss(tp); return left <= tcp_max_tso_deferred_mss(tp);
} }
......
...@@ -1522,21 +1522,21 @@ static void tcp_cwnd_validate(struct sock *sk) ...@@ -1522,21 +1522,21 @@ static void tcp_cwnd_validate(struct sock *sk)
* when we would be allowed to send the split-due-to-Nagle skb fully. * when we would be allowed to send the split-due-to-Nagle skb fully.
*/ */
static unsigned int tcp_mss_split_point(const struct sock *sk, const struct sk_buff *skb, static unsigned int tcp_mss_split_point(const struct sock *sk, const struct sk_buff *skb,
unsigned int mss_now, unsigned int cwnd) unsigned int mss_now, unsigned int max_segs)
{ {
const struct tcp_sock *tp = tcp_sk(sk); const struct tcp_sock *tp = tcp_sk(sk);
u32 needed, window, cwnd_len; u32 needed, window, max_len;
window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
cwnd_len = mss_now * cwnd; max_len = mss_now * max_segs;
if (likely(cwnd_len <= window && skb != tcp_write_queue_tail(sk))) if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
return cwnd_len; return max_len;
needed = min(skb->len, window); needed = min(skb->len, window);
if (cwnd_len <= needed) if (max_len <= needed)
return cwnd_len; return max_len;
return needed - needed % mss_now; return needed - needed % mss_now;
} }
...@@ -1765,7 +1765,8 @@ static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb) ...@@ -1765,7 +1765,8 @@ static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
limit = min(send_win, cong_win); limit = min(send_win, cong_win);
/* If a full-sized TSO skb can be sent, do it. */ /* If a full-sized TSO skb can be sent, do it. */
if (limit >= sk->sk_gso_max_size) if (limit >= min_t(unsigned int, sk->sk_gso_max_size,
sk->sk_gso_max_segs * tp->mss_cache))
goto send_now; goto send_now;
/* Middle in queue won't get any more data, full sendable already? */ /* Middle in queue won't get any more data, full sendable already? */
...@@ -1999,7 +2000,9 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, ...@@ -1999,7 +2000,9 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
limit = mss_now; limit = mss_now;
if (tso_segs > 1 && !tcp_urg_mode(tp)) if (tso_segs > 1 && !tcp_urg_mode(tp))
limit = tcp_mss_split_point(sk, skb, mss_now, limit = tcp_mss_split_point(sk, skb, mss_now,
cwnd_quota); min_t(unsigned int,
cwnd_quota,
sk->sk_gso_max_segs));
if (skb->len > limit && if (skb->len > limit &&
unlikely(tso_fragment(sk, skb, limit, mss_now, gfp))) unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment