Commit fdd60d80 authored by Willem de Bruijn's avatar Willem de Bruijn Committed by Greg Kroah-Hartman

udp: correct reuseport selection with connected sockets

[ Upstream commit acdcecc6 ]

UDP reuseport groups can hold a mix unconnected and connected sockets.
Ensure that connections only receive all traffic to their 4-tuple.

Fast reuseport returns on the first reuseport match on the assumption
that all matches are equal. Only if connections are present, return to
the previous behavior of scoring all sockets.

Record if connections are present and if so (1) treat such connected
sockets as an independent match from the group, (2) only return
2-tuple matches from reuseport and (3) do not return on the first
2-tuple reuseport match to allow for a higher scoring match later.

New field has_conns is set without locks. No other fields in the
bitmap are modified at runtime and the field is only ever set
unconditionally, so an RMW cannot miss a change.

Fixes: e32ea7e7 ("soreuseport: fast reuseport UDP socket selection")
Link: http://lkml.kernel.org/r/CA+FuTSfRP09aJNYRt04SS6qj22ViiOEWaWmLAwX0psk8-PGNxw@mail.gmail.comSigned-off-by: default avatarWillem de Bruijn <willemb@google.com>
Acked-by: default avatarPaolo Abeni <pabeni@redhat.com>
Acked-by: default avatarCraig Gallek <kraig@google.com>
Signed-off-by: default avatarWillem de Bruijn <willemb@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 97b5f8c9
...@@ -21,7 +21,8 @@ struct sock_reuseport { ...@@ -21,7 +21,8 @@ struct sock_reuseport {
unsigned int synq_overflow_ts; unsigned int synq_overflow_ts;
/* ID stays the same even after the size of socks[] grows. */ /* ID stays the same even after the size of socks[] grows. */
unsigned int reuseport_id; unsigned int reuseport_id;
bool bind_inany; unsigned int bind_inany:1;
unsigned int has_conns:1;
struct bpf_prog __rcu *prog; /* optional BPF sock selector */ struct bpf_prog __rcu *prog; /* optional BPF sock selector */
struct sock *socks[0]; /* array of sock pointers */ struct sock *socks[0]; /* array of sock pointers */
}; };
...@@ -35,6 +36,24 @@ extern struct sock *reuseport_select_sock(struct sock *sk, ...@@ -35,6 +36,24 @@ extern struct sock *reuseport_select_sock(struct sock *sk,
struct sk_buff *skb, struct sk_buff *skb,
int hdr_len); int hdr_len);
extern int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog); extern int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog);
static inline bool reuseport_has_conns(struct sock *sk, bool set)
{
struct sock_reuseport *reuse;
bool ret = false;
rcu_read_lock();
reuse = rcu_dereference(sk->sk_reuseport_cb);
if (reuse) {
if (set)
reuse->has_conns = 1;
ret = reuse->has_conns;
}
rcu_read_unlock();
return ret;
}
int reuseport_get_id(struct sock_reuseport *reuse); int reuseport_get_id(struct sock_reuseport *reuse);
#endif /* _SOCK_REUSEPORT_H */ #endif /* _SOCK_REUSEPORT_H */
...@@ -292,8 +292,19 @@ struct sock *reuseport_select_sock(struct sock *sk, ...@@ -292,8 +292,19 @@ struct sock *reuseport_select_sock(struct sock *sk,
select_by_hash: select_by_hash:
/* no bpf or invalid bpf result: fall back to hash usage */ /* no bpf or invalid bpf result: fall back to hash usage */
if (!sk2) if (!sk2) {
sk2 = reuse->socks[reciprocal_scale(hash, socks)]; int i, j;
i = j = reciprocal_scale(hash, socks);
while (reuse->socks[i]->sk_state == TCP_ESTABLISHED) {
i++;
if (i >= reuse->num_socks)
i = 0;
if (i == j)
goto out;
}
sk2 = reuse->socks[i];
}
} }
out: out:
......
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
#include <net/sock.h> #include <net/sock.h>
#include <net/route.h> #include <net/route.h>
#include <net/tcp_states.h> #include <net/tcp_states.h>
#include <net/sock_reuseport.h>
int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
{ {
...@@ -73,6 +74,7 @@ int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len ...@@ -73,6 +74,7 @@ int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len
} }
inet->inet_daddr = fl4->daddr; inet->inet_daddr = fl4->daddr;
inet->inet_dport = usin->sin_port; inet->inet_dport = usin->sin_port;
reuseport_has_conns(sk, true);
sk->sk_state = TCP_ESTABLISHED; sk->sk_state = TCP_ESTABLISHED;
sk_set_txhash(sk); sk_set_txhash(sk);
inet->inet_id = jiffies; inet->inet_id = jiffies;
......
...@@ -443,12 +443,13 @@ static struct sock *udp4_lib_lookup2(struct net *net, ...@@ -443,12 +443,13 @@ static struct sock *udp4_lib_lookup2(struct net *net,
score = compute_score(sk, net, saddr, sport, score = compute_score(sk, net, saddr, sport,
daddr, hnum, dif, sdif, exact_dif); daddr, hnum, dif, sdif, exact_dif);
if (score > badness) { if (score > badness) {
if (sk->sk_reuseport) { if (sk->sk_reuseport &&
sk->sk_state != TCP_ESTABLISHED) {
hash = udp_ehashfn(net, daddr, hnum, hash = udp_ehashfn(net, daddr, hnum,
saddr, sport); saddr, sport);
result = reuseport_select_sock(sk, hash, skb, result = reuseport_select_sock(sk, hash, skb,
sizeof(struct udphdr)); sizeof(struct udphdr));
if (result) if (result && !reuseport_has_conns(sk, false))
return result; return result;
} }
badness = score; badness = score;
......
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include <net/ip6_route.h> #include <net/ip6_route.h>
#include <net/tcp_states.h> #include <net/tcp_states.h>
#include <net/dsfield.h> #include <net/dsfield.h>
#include <net/sock_reuseport.h>
#include <linux/errqueue.h> #include <linux/errqueue.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
...@@ -258,6 +259,7 @@ int __ip6_datagram_connect(struct sock *sk, struct sockaddr *uaddr, ...@@ -258,6 +259,7 @@ int __ip6_datagram_connect(struct sock *sk, struct sockaddr *uaddr,
goto out; goto out;
} }
reuseport_has_conns(sk, true);
sk->sk_state = TCP_ESTABLISHED; sk->sk_state = TCP_ESTABLISHED;
sk_set_txhash(sk); sk_set_txhash(sk);
out: out:
......
...@@ -177,13 +177,14 @@ static struct sock *udp6_lib_lookup2(struct net *net, ...@@ -177,13 +177,14 @@ static struct sock *udp6_lib_lookup2(struct net *net,
score = compute_score(sk, net, saddr, sport, score = compute_score(sk, net, saddr, sport,
daddr, hnum, dif, sdif, exact_dif); daddr, hnum, dif, sdif, exact_dif);
if (score > badness) { if (score > badness) {
if (sk->sk_reuseport) { if (sk->sk_reuseport &&
sk->sk_state != TCP_ESTABLISHED) {
hash = udp6_ehashfn(net, daddr, hnum, hash = udp6_ehashfn(net, daddr, hnum,
saddr, sport); saddr, sport);
result = reuseport_select_sock(sk, hash, skb, result = reuseport_select_sock(sk, hash, skb,
sizeof(struct udphdr)); sizeof(struct udphdr));
if (result) if (result && !reuseport_has_conns(sk, false))
return result; return result;
} }
result = sk; result = sk;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment