Commit fe4f70cf authored by Alexey Kuznetsov's avatar Alexey Kuznetsov Committed by David S. Miller

[NET]: Add segmentation offload support to TCP.

parent d35cb51f
......@@ -241,7 +241,8 @@ struct tcp_opt {
__u32 snd_wnd; /* The window we expect to receive */
__u32 max_window; /* Maximal window ever seen from peer */
__u32 pmtu_cookie; /* Last pmtu seen by socket */
__u16 mss_cache; /* Cached effective mss, not including SACKS */
__u32 mss_cache; /* Cached effective mss, not including SACKS */
__u16 mss_cache_std; /* Like mss_cache, but without TSO */
__u16 mss_clamp; /* Maximal mss, negotiated at connection setup */
__u16 ext_header_len; /* Network protocol overhead (IP/IPv6 options) */
__u8 ca_state; /* State of fast-retransmit machine */
......
......@@ -53,12 +53,13 @@ static inline void inet_putpeer(struct inet_peer *p)
extern spinlock_t inet_peer_idlock;
/* can be called with or without local BH being disabled */
static inline __u16 inet_getid(struct inet_peer *p)
static inline __u16 inet_getid(struct inet_peer *p, int more)
{
__u16 id;
spin_lock_bh(&inet_peer_idlock);
id = p->ip_id_count++;
id = p->ip_id_count;
p->ip_id_count += 1 + more;
spin_unlock_bh(&inet_peer_idlock);
return id;
}
......
......@@ -187,7 +187,7 @@ int ip_dont_fragment(struct sock *sk, struct dst_entry *dst)
!(dst->mxlock&(1<<RTAX_MTU))));
}
extern void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst);
extern void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more);
static inline void ip_select_ident(struct iphdr *iph, struct dst_entry *dst, struct sock *sk)
{
......@@ -200,7 +200,19 @@ static inline void ip_select_ident(struct iphdr *iph, struct dst_entry *dst, str
iph->id = (sk && inet_sk(sk)->daddr) ?
htons(inet_sk(sk)->id++) : 0;
} else
__ip_select_ident(iph, dst);
__ip_select_ident(iph, dst, 0);
}
static inline void ip_select_ident_more(struct iphdr *iph, struct dst_entry *dst, struct sock *sk, int more)
{
if (iph->frag_off&__constant_htons(IP_DF)) {
if (sk && inet_sk(sk)->daddr) {
iph->id = htons(inet_sk(sk)->id);
inet_sk(sk)->id += 1 + more;
} else
iph->id = 0;
} else
__ip_select_ident(iph, dst, more);
}
/*
......
......@@ -130,7 +130,7 @@ struct sock {
bsdism;
unsigned char debug;
unsigned char rcvtstamp;
/* Hole of 1 byte. Try to pack. */
unsigned char no_largesend;
int route_caps;
int proc;
unsigned long lingertime;
......
......@@ -905,13 +905,18 @@ static inline void tcp_reset_xmit_timer(struct sock *sk, int what, unsigned long
/* Compute the current effective MSS, taking SACKs and IP options,
* and even PMTU discovery events into account.
*
* LARGESEND note: !urg_mode is overkill, only frames up to snd_up
* cannot be large. However, taking into account rare use of URG, this
* is not a big flaw.
*/
static __inline__ unsigned int tcp_current_mss(struct sock *sk)
static __inline__ unsigned int tcp_current_mss(struct sock *sk, int large)
{
struct tcp_opt *tp = tcp_sk(sk);
struct dst_entry *dst = __sk_dst_get(sk);
int mss_now = tp->mss_cache;
int mss_now = large && (sk->route_caps&NETIF_F_TSO) && !tp->urg_mode ?
tp->mss_cache : tp->mss_cache_std;
if (dst && dst->pmtu != tp->pmtu_cookie)
mss_now = tcp_sync_mss(sk, dst->pmtu);
......@@ -933,7 +938,7 @@ static __inline__ unsigned int tcp_current_mss(struct sock *sk)
static inline void tcp_initialize_rcv_mss(struct sock *sk)
{
struct tcp_opt *tp = tcp_sk(sk);
unsigned int hint = min(tp->advmss, tp->mss_cache);
unsigned int hint = min(tp->advmss, tp->mss_cache_std);
hint = min(hint, tp->rcv_wnd/2);
hint = min(hint, TCP_MIN_RCVMSS);
......@@ -1269,7 +1274,7 @@ static __inline__ void __tcp_push_pending_frames(struct sock *sk,
static __inline__ void tcp_push_pending_frames(struct sock *sk,
struct tcp_opt *tp)
{
__tcp_push_pending_frames(sk, tp, tcp_current_mss(sk), tp->nonagle);
__tcp_push_pending_frames(sk, tp, tcp_current_mss(sk, 1), tp->nonagle);
}
static __inline__ int tcp_may_send_now(struct sock *sk, struct tcp_opt *tp)
......@@ -1277,7 +1282,7 @@ static __inline__ int tcp_may_send_now(struct sock *sk, struct tcp_opt *tp)
struct sk_buff *skb = tp->send_head;
return (skb &&
tcp_snd_test(tp, skb, tcp_current_mss(sk),
tcp_snd_test(tp, skb, tcp_current_mss(sk, 1),
tcp_skb_is_last(sk, skb) ? 1 : tp->nonagle));
}
......@@ -1839,6 +1844,15 @@ static inline int tcp_paws_check(struct tcp_opt *tp, int rst)
return 1;
}
static inline void tcp_v4_setup_caps(struct sock *sk, struct dst_entry *dst)
{
sk->route_caps = dst->dev->features;
if (sk->route_caps & NETIF_F_TSO) {
if (sk->no_largesend)
sk->route_caps &= ~NETIF_F_TSO;
}
}
#define TCP_CHECK_TIMER(sk) do { } while (0)
#endif /* _TCP_H */
......@@ -28,12 +28,13 @@ TCP_ECN_send_synack(struct tcp_opt *tp, struct sk_buff *skb)
}
static __inline__ void
TCP_ECN_send_syn(struct tcp_opt *tp, struct sk_buff *skb)
TCP_ECN_send_syn(struct sock *sk, struct tcp_opt *tp, struct sk_buff *skb)
{
tp->ecn_flags = 0;
if (sysctl_tcp_ecn) {
if (sysctl_tcp_ecn && !(sk->route_caps&NETIF_F_TSO)) {
TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_ECE|TCPCB_FLAG_CWR;
tp->ecn_flags = TCP_ECN_OK;
sk->no_largesend = 1;
}
}
......
......@@ -306,10 +306,20 @@ static inline int ip_queue_xmit2(struct sk_buff *skb)
iph = skb->nh.iph;
}
if (skb->len > rt->u.dst.pmtu)
if (skb->len > rt->u.dst.pmtu) {
unsigned int hlen;
if (!(sk->route_caps&NETIF_F_TSO))
goto fragment;
ip_select_ident(iph, &rt->u.dst, sk);
/* Hack zone: all this must be done by TCP. */
hlen = ((skb->h.raw - skb->data) + (skb->h.th->doff << 2));
skb_shinfo(skb)->tso_size = rt->u.dst.pmtu - hlen;
skb_shinfo(skb)->tso_segs =
(skb->len - hlen + skb_shinfo(skb)->tso_size - 1)/
skb_shinfo(skb)->tso_size - 1;
}
ip_select_ident_more(iph, &rt->u.dst, sk, skb_shinfo(skb)->tso_segs);
/* Add an IP checksum. */
ip_send_check(iph);
......@@ -371,7 +381,7 @@ int ip_queue_xmit(struct sk_buff *skb)
sk->bound_dev_if))
goto no_route;
__sk_dst_set(sk, &rt->u.dst);
sk->route_caps = rt->u.dst.dev->features;
tcp_v4_setup_caps(sk, &rt->u.dst);
}
skb->dst = dst_clone(&rt->u.dst);
......@@ -577,7 +587,7 @@ static int ip_build_xmit_slow(struct sock *sk,
* for packets without DF or having
* been fragmented.
*/
__ip_select_ident(iph, &rt->u.dst);
__ip_select_ident(iph, &rt->u.dst, 0);
id = iph->id;
}
......
......@@ -729,7 +729,7 @@ static void ip_select_fb_ident(struct iphdr *iph)
spin_unlock_bh(&ip_fb_id_lock);
}
void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst)
void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
{
struct rtable *rt = (struct rtable *) dst;
......@@ -741,7 +741,7 @@ void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst)
so that we need not to grab a lock to dereference it.
*/
if (rt->peer) {
iph->id = htons(inet_getid(rt->peer));
iph->id = htons(inet_getid(rt->peer, more));
return;
}
} else
......
......@@ -846,7 +846,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset,
clear_bit(SOCK_ASYNC_NOSPACE, &sk->socket->flags);
mss_now = tcp_current_mss(sk);
mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
copied = 0;
err = -EPIPE;
......@@ -921,7 +921,7 @@ ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset,
if ((err = wait_for_tcp_memory(sk, &timeo)) != 0)
goto do_error;
mss_now = tcp_current_mss(sk);
mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
}
out:
......@@ -1001,7 +1001,7 @@ static inline int skb_add_data(struct sk_buff *skb, char *from, int copy)
static inline int select_size(struct sock *sk, struct tcp_opt *tp)
{
int tmp = tp->mss_cache;
int tmp = tp->mss_cache_std;
if (sk->route_caps & NETIF_F_SG) {
int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
......@@ -1037,7 +1037,7 @@ int tcp_sendmsg(struct sock *sk, struct msghdr *msg, int size)
/* This should be in poll */
clear_bit(SOCK_ASYNC_NOSPACE, &sk->socket->flags);
mss_now = tcp_current_mss(sk);
mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
/* Ok commence sending. */
iovlen = msg->msg_iovlen;
......@@ -1192,7 +1192,7 @@ int tcp_sendmsg(struct sock *sk, struct msghdr *msg, int size)
if ((err = wait_for_tcp_memory(sk, &timeo)) != 0)
goto do_error;
mss_now = tcp_current_mss(sk);
mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
}
}
......@@ -2444,7 +2444,7 @@ int tcp_getsockopt(struct sock *sk, int level, int optname, char *optval,
switch (optname) {
case TCP_MAXSEG:
val = tp->mss_cache;
val = tp->mss_cache_std;
if (!val && ((1 << sk->state) & (TCPF_CLOSE | TCPF_LISTEN)))
val = tp->user_mss;
break;
......@@ -2507,7 +2507,7 @@ int tcp_getsockopt(struct sock *sk, int level, int optname, char *optval,
info.tcpi_rto = (1000000 * tp->rto) / HZ;
info.tcpi_ato = (1000000 * tp->ack.ato) / HZ;
info.tcpi_snd_mss = tp->mss_cache;
info.tcpi_snd_mss = tp->mss_cache_std;
info.tcpi_rcv_mss = tp->ack.rcv_mss;
info.tcpi_unacked = tp->packets_out;
......
......@@ -772,6 +772,14 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_
int flag = 0;
int i;
/* So, SACKs for already sent large segments will be lost.
* Not good, but alternative is to resegment the queue. */
if (sk->route_caps&NETIF_F_TSO) {
sk->route_caps &= ~NETIF_F_TSO;
sk->no_largesend = 1;
tp->mss_cache = tp->mss_cache_std;
}
if (!tp->sacked_out)
tp->fackets_out = 0;
prior_fackets = tp->fackets_out;
......@@ -2963,6 +2971,8 @@ void tcp_cwnd_application_limited(struct sock *sk)
/* When incoming ACK allowed to free some skb from write_queue,
* we remember this event in flag tp->queue_shrunk and wake up socket
* on the exit from tcp input handler.
*
* PROBLEM: sndbuf expansion does not work well with largesend.
*/
static void tcp_new_space(struct sock *sk)
{
......@@ -2972,8 +2982,8 @@ static void tcp_new_space(struct sock *sk)
!(sk->userlocks&SOCK_SNDBUF_LOCK) &&
!tcp_memory_pressure &&
atomic_read(&tcp_memory_allocated) < sysctl_tcp_mem[0]) {
int sndmem = tp->mss_clamp + MAX_TCP_HEADER + 16 +
sizeof(struct sk_buff),
int sndmem = max_t(u32, tp->mss_clamp, tp->mss_cache) +
MAX_TCP_HEADER + 16 + sizeof(struct sk_buff),
demanded = max_t(unsigned int, tp->snd_cwnd,
tp->reordering + 1);
sndmem *= 2*demanded;
......@@ -3502,6 +3512,8 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
*/
TCP_ECN_rcv_synack(tp, th);
if (tp->ecn_flags&TCP_ECN_OK)
sk->no_largesend = 1;
tp->snd_wl1 = TCP_SKB_CB(skb)->seq;
tcp_ack(sk, skb, FLAG_SLOWPATH);
......@@ -3627,10 +3639,13 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
tp->snd_wl1 = TCP_SKB_CB(skb)->seq;
tp->max_window = tp->snd_wnd;
TCP_ECN_rcv_syn(tp, th);
if (tp->ecn_flags&TCP_ECN_OK)
sk->no_largesend = 1;
tcp_sync_mss(sk, tp->pmtu_cookie);
tcp_initialize_rcv_mss(sk);
TCP_ECN_rcv_syn(tp, th);
tcp_send_synack(sk);
#if 0
......
......@@ -780,7 +780,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
}
__sk_dst_set(sk, &rt->u.dst);
sk->route_caps = rt->u.dst.dev->features;
tcp_v4_setup_caps(sk, &rt->u.dst);
if (!inet->opt || !inet->opt->srr)
daddr = rt->rt_dst;
......@@ -1559,7 +1559,7 @@ struct sock *tcp_v4_syn_recv_sock(struct sock *sk, struct sk_buff *skb,
goto exit;
newsk->dst_cache = dst;
newsk->route_caps = dst->dev->features;
tcp_v4_setup_caps(newsk, dst);
newtp = tcp_sk(newsk);
newinet = inet_sk(newsk);
......@@ -1865,7 +1865,7 @@ static int tcp_v4_reselect_saddr(struct sock *sk)
return err;
__sk_dst_set(sk, &rt->u.dst);
sk->route_caps = rt->u.dst.dev->features;
tcp_v4_setup_caps(sk, &rt->u.dst);
new_saddr = rt->rt_src;
......@@ -1913,7 +1913,7 @@ int tcp_v4_rebuild_header(struct sock *sk)
RT_CONN_FLAGS(sk), sk->bound_dev_if);
if (!err) {
__sk_dst_set(sk, &rt->u.dst);
sk->route_caps = rt->u.dst.dev->features;
tcp_v4_setup_caps(sk, &rt->u.dst);
return 0;
}
......
......@@ -786,6 +786,8 @@ struct sock *tcp_create_openreq_child(struct sock *sk, struct open_request *req,
newtp->ack.last_seg_size = skb->len-newtp->tcp_header_len;
newtp->mss_clamp = req->mss;
TCP_ECN_openreq_child(newtp, req);
if (newtp->ecn_flags&TCP_ECN_OK)
newsk->no_largesend = 1;
TCP_INC_STATS_BH(TcpPassiveOpens);
}
......
......@@ -531,7 +531,21 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu)
/* And store cached results */
tp->pmtu_cookie = pmtu;
tp->mss_cache = mss_now;
tp->mss_cache = tp->mss_cache_std = mss_now;
if (sk->route_caps&NETIF_F_TSO) {
int large_mss;
large_mss = 65535 - tp->af_specific->net_header_len -
tp->ext_header_len - tp->tcp_header_len;
if (tp->max_window && large_mss > (tp->max_window>>1))
large_mss = max((tp->max_window>>1), 68U - tp->tcp_header_len);
/* Always keep large mss multiple of real mss. */
tp->mss_cache = mss_now*(large_mss/mss_now);
}
return mss_now;
}
......@@ -561,7 +575,7 @@ int tcp_write_xmit(struct sock *sk, int nonagle)
* We also handle things correctly when the user adds some
* IP options mid-stream. Silly to do, but cover it.
*/
mss_now = tcp_current_mss(sk);
mss_now = tcp_current_mss(sk, 1);
while((skb = tp->send_head) &&
tcp_snd_test(tp, skb, mss_now, tcp_skb_is_last(sk, skb) ? nonagle : 1)) {
......@@ -767,7 +781,7 @@ void tcp_simple_retransmit(struct sock *sk)
{
struct tcp_opt *tp = tcp_sk(sk);
struct sk_buff *skb;
unsigned int mss = tcp_current_mss(sk);
unsigned int mss = tcp_current_mss(sk, 0);
int lost = 0;
for_retrans_queue(skb, sk, tp) {
......@@ -812,7 +826,7 @@ void tcp_simple_retransmit(struct sock *sk)
int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
{
struct tcp_opt *tp = tcp_sk(sk);
unsigned int cur_mss = tcp_current_mss(sk);
unsigned int cur_mss = tcp_current_mss(sk, 0);
int err;
/* Do not sent more than we queued. 1/4 is reserved for possible
......@@ -821,6 +835,27 @@ int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
if (atomic_read(&sk->wmem_alloc) > min(sk->wmem_queued+(sk->wmem_queued>>2),sk->sndbuf))
return -EAGAIN;
if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
struct sk_buff *skb2;
if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
BUG();
if (sk->route_caps&NETIF_F_TSO) {
sk->route_caps &= ~NETIF_F_TSO;
sk->no_largesend = 1;
tp->mss_cache = tp->mss_cache_std;
}
if(tcp_fragment(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
return -ENOMEM;
skb2 = skb->next;
__skb_unlink(skb, skb->list);
tcp_free_skb(sk, skb);
skb = skb2;
}
/* If receiver has shrunk his window, and skb is out of
* new window, do not retransmit it. The exception is the
* case, when window is shrunk to zero. In this case
......@@ -998,7 +1033,7 @@ void tcp_send_fin(struct sock *sk)
* unsent frames. But be careful about outgoing SACKS
* and IP options.
*/
mss_now = tcp_current_mss(sk);
mss_now = tcp_current_mss(sk, 1);
if(tp->send_head != NULL) {
TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN;
......@@ -1121,6 +1156,8 @@ struct sk_buff * tcp_make_synack(struct sock *sk, struct dst_entry *dst,
memset(th, 0, sizeof(struct tcphdr));
th->syn = 1;
th->ack = 1;
if (dst->dev->features&NETIF_F_TSO)
req->ecn_ok = 0;
TCP_ECN_make_synack(req, th);
th->source = inet_sk(sk)->sport;
th->dest = req->rmt_port;
......@@ -1224,7 +1261,7 @@ int tcp_connect(struct sock *sk)
skb_reserve(buff, MAX_TCP_HEADER);
TCP_SKB_CB(buff)->flags = TCPCB_FLAG_SYN;
TCP_ECN_send_syn(tp, buff);
TCP_ECN_send_syn(sk, tp, buff);
TCP_SKB_CB(buff)->sacked = 0;
buff->csum = 0;
TCP_SKB_CB(buff)->seq = tp->write_seq++;
......@@ -1379,7 +1416,7 @@ int tcp_write_wakeup(struct sock *sk)
if ((skb = tp->send_head) != NULL &&
before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)) {
int err;
int mss = tcp_current_mss(sk);
int mss = tcp_current_mss(sk, 0);
int seg_size = tp->snd_una+tp->snd_wnd-TCP_SKB_CB(skb)->seq;
if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
......@@ -1395,6 +1432,13 @@ int tcp_write_wakeup(struct sock *sk)
TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
if (tcp_fragment(sk, skb, seg_size))
return -1;
/* SWS override triggered forced fragmentation.
* Disable TSO, the connection is too sick. */
if (sk->route_caps&NETIF_F_TSO) {
sk->no_largesend = 1;
sk->route_caps &= ~NETIF_F_TSO;
tp->mss_cache = tp->mss_cache_std;
}
}
TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
TCP_SKB_CB(skb)->when = tcp_time_stamp;
......
......@@ -659,7 +659,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
}
ip6_dst_store(sk, dst, NULL);
sk->route_caps = dst->dev->features&~NETIF_F_IP_CSUM;
sk->route_caps = dst->dev->features&~(NETIF_F_IP_CSUM|NETIF_F_TSO);
if (saddr == NULL) {
err = ipv6_get_saddr(dst, &np->daddr, &saddr_buf);
......@@ -1333,7 +1333,7 @@ static struct sock * tcp_v6_syn_recv_sock(struct sock *sk, struct sk_buff *skb,
MOD_INC_USE_COUNT;
ip6_dst_store(newsk, dst, NULL);
sk->route_caps = dst->dev->features&~NETIF_F_IP_CSUM;
sk->route_caps = dst->dev->features&~(NETIF_F_IP_CSUM|NETIF_F_TSO);
newtcp6sk = (struct tcp6_sock *)newsk;
newtcp6sk->pinet6 = &newtcp6sk->inet6;
......@@ -1721,7 +1721,7 @@ static int tcp_v6_rebuild_header(struct sock *sk)
}
ip6_dst_store(sk, dst, NULL);
sk->route_caps = dst->dev->features&~NETIF_F_IP_CSUM;
sk->route_caps = dst->dev->features&~(NETIF_F_IP_CSUM|NETIF_F_TSO);
}
return 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment