Commit 271b72c7 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

udp: RCU handling for Unicast packets.

Goals are :

1) Optimizing handling of incoming Unicast UDP frames, so that no memory
 writes should happen in the fast path.

 Note: Multicasts and broadcasts still will need to take a lock,
 because doing a full lockless lookup in this case is difficult.

2) No expensive operations in the socket bind/unhash phases :
  - No expensive synchronize_rcu() calls.

  - No added rcu_head in socket structure, increasing memory needs,
  but more important, forcing us to use call_rcu() calls,
  that have the bad property of making sockets structure cold.
  (rcu grace period between socket freeing and its potential reuse
   make this socket being cold in CPU cache).
  David did a previous patch using call_rcu() and noticed a 20%
  impact on TCP connection rates.
  Quoting Cristopher Lameter :
   "Right. That results in cacheline cooldown. You'd want to recycle
    the object as they are cache hot on a per cpu basis. That is screwed
    up by the delayed regular rcu processing. We have seen multiple
    regressions due to cacheline cooldown.
    The only choice in cacheline hot sensitive areas is to deal with the
    complexity that comes with SLAB_DESTROY_BY_RCU or give up on RCU."

  - Because udp sockets are allocated from dedicated kmem_cache,
  use of SLAB_DESTROY_BY_RCU can help here.

Theory of operation :
---------------------

As the lookup is lockfree (using rcu_read_lock()/rcu_read_unlock()),
special attention must be taken by readers and writers.

Use of SLAB_DESTROY_BY_RCU is tricky too, because a socket can be freed,
reused, inserted in a different chain or in worst case in the same chain
while readers could do lookups in the same time.

In order to avoid loops, a reader must check each socket found in a chain
really belongs to the chain the reader was traversing. If it finds a
mismatch, lookup must start again at the begining. This *restart* loop
is the reason we had to use rdlock for the multicast case, because
we dont want to send same message several times to the same socket.

We use RCU only for fast path.
Thus, /proc/net/udp still takes spinlocks.
Signed-off-by: default avatarEric Dumazet <dada1@cosmosbay.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 645ca708
...@@ -363,6 +363,27 @@ static __inline__ int sk_del_node_init(struct sock *sk) ...@@ -363,6 +363,27 @@ static __inline__ int sk_del_node_init(struct sock *sk)
return rc; return rc;
} }
static __inline__ int __sk_del_node_init_rcu(struct sock *sk)
{
if (sk_hashed(sk)) {
hlist_del_init_rcu(&sk->sk_node);
return 1;
}
return 0;
}
static __inline__ int sk_del_node_init_rcu(struct sock *sk)
{
int rc = __sk_del_node_init_rcu(sk);
if (rc) {
/* paranoid for a while -acme */
WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
__sock_put(sk);
}
return rc;
}
static __inline__ void __sk_add_node(struct sock *sk, struct hlist_head *list) static __inline__ void __sk_add_node(struct sock *sk, struct hlist_head *list)
{ {
hlist_add_head(&sk->sk_node, list); hlist_add_head(&sk->sk_node, list);
...@@ -374,6 +395,17 @@ static __inline__ void sk_add_node(struct sock *sk, struct hlist_head *list) ...@@ -374,6 +395,17 @@ static __inline__ void sk_add_node(struct sock *sk, struct hlist_head *list)
__sk_add_node(sk, list); __sk_add_node(sk, list);
} }
static __inline__ void __sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
{
hlist_add_head_rcu(&sk->sk_node, list);
}
static __inline__ void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
{
sock_hold(sk);
__sk_add_node_rcu(sk, list);
}
static __inline__ void __sk_del_bind_node(struct sock *sk) static __inline__ void __sk_del_bind_node(struct sock *sk)
{ {
__hlist_del(&sk->sk_bind_node); __hlist_del(&sk->sk_bind_node);
...@@ -387,6 +419,8 @@ static __inline__ void sk_add_bind_node(struct sock *sk, ...@@ -387,6 +419,8 @@ static __inline__ void sk_add_bind_node(struct sock *sk,
#define sk_for_each(__sk, node, list) \ #define sk_for_each(__sk, node, list) \
hlist_for_each_entry(__sk, node, list, sk_node) hlist_for_each_entry(__sk, node, list, sk_node)
#define sk_for_each_rcu(__sk, node, list) \
hlist_for_each_entry_rcu(__sk, node, list, sk_node)
#define sk_for_each_from(__sk, node) \ #define sk_for_each_from(__sk, node) \
if (__sk && ({ node = &(__sk)->sk_node; 1; })) \ if (__sk && ({ node = &(__sk)->sk_node; 1; })) \
hlist_for_each_entry_from(__sk, node, sk_node) hlist_for_each_entry_from(__sk, node, sk_node)
...@@ -589,8 +623,9 @@ struct proto { ...@@ -589,8 +623,9 @@ struct proto {
int *sysctl_rmem; int *sysctl_rmem;
int max_header; int max_header;
struct kmem_cache *slab; struct kmem_cache *slab;
unsigned int obj_size; unsigned int obj_size;
int slab_flags;
atomic_t *orphan_count; atomic_t *orphan_count;
......
...@@ -2042,7 +2042,8 @@ int proto_register(struct proto *prot, int alloc_slab) ...@@ -2042,7 +2042,8 @@ int proto_register(struct proto *prot, int alloc_slab)
if (alloc_slab) { if (alloc_slab) {
prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0, prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0,
SLAB_HWCACHE_ALIGN, NULL); SLAB_HWCACHE_ALIGN | prot->slab_flags,
NULL);
if (prot->slab == NULL) { if (prot->slab == NULL) {
printk(KERN_CRIT "%s: Can't create sock SLAB cache!\n", printk(KERN_CRIT "%s: Can't create sock SLAB cache!\n",
......
...@@ -187,7 +187,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, ...@@ -187,7 +187,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum,
inet_sk(sk)->num = snum; inet_sk(sk)->num = snum;
sk->sk_hash = snum; sk->sk_hash = snum;
if (sk_unhashed(sk)) { if (sk_unhashed(sk)) {
sk_add_node(sk, &hslot->head); sk_add_node_rcu(sk, &hslot->head);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
} }
error = 0; error = 0;
...@@ -253,15 +253,24 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, ...@@ -253,15 +253,24 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
__be16 sport, __be32 daddr, __be16 dport, __be16 sport, __be32 daddr, __be16 dport,
int dif, struct udp_table *udptable) int dif, struct udp_table *udptable)
{ {
struct sock *sk, *result = NULL; struct sock *sk, *result;
struct hlist_node *node; struct hlist_node *node;
unsigned short hnum = ntohs(dport); unsigned short hnum = ntohs(dport);
unsigned int hash = udp_hashfn(net, hnum); unsigned int hash = udp_hashfn(net, hnum);
struct udp_hslot *hslot = &udptable->hash[hash]; struct udp_hslot *hslot = &udptable->hash[hash];
int score, badness = -1; int score, badness;
spin_lock(&hslot->lock); rcu_read_lock();
sk_for_each(sk, node, &hslot->head) { begin:
result = NULL;
badness = -1;
sk_for_each_rcu(sk, node, &hslot->head) {
/*
* lockless reader, and SLAB_DESTROY_BY_RCU items:
* We must check this item was not moved to another chain
*/
if (udp_hashfn(net, sk->sk_hash) != hash)
goto begin;
score = compute_score(sk, net, saddr, hnum, sport, score = compute_score(sk, net, saddr, hnum, sport,
daddr, dport, dif); daddr, dport, dif);
if (score > badness) { if (score > badness) {
...@@ -269,9 +278,16 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, ...@@ -269,9 +278,16 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
badness = score; badness = score;
} }
} }
if (result) if (result) {
sock_hold(result); if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
spin_unlock(&hslot->lock); result = NULL;
else if (unlikely(compute_score(result, net, saddr, hnum, sport,
daddr, dport, dif) < badness)) {
sock_put(result);
goto begin;
}
}
rcu_read_unlock();
return result; return result;
} }
...@@ -953,7 +969,7 @@ void udp_lib_unhash(struct sock *sk) ...@@ -953,7 +969,7 @@ void udp_lib_unhash(struct sock *sk)
struct udp_hslot *hslot = &udptable->hash[hash]; struct udp_hslot *hslot = &udptable->hash[hash];
spin_lock(&hslot->lock); spin_lock(&hslot->lock);
if (sk_del_node_init(sk)) { if (sk_del_node_init_rcu(sk)) {
inet_sk(sk)->num = 0; inet_sk(sk)->num = 0;
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
} }
...@@ -1517,6 +1533,7 @@ struct proto udp_prot = { ...@@ -1517,6 +1533,7 @@ struct proto udp_prot = {
.sysctl_wmem = &sysctl_udp_wmem_min, .sysctl_wmem = &sysctl_udp_wmem_min,
.sysctl_rmem = &sysctl_udp_rmem_min, .sysctl_rmem = &sysctl_udp_rmem_min,
.obj_size = sizeof(struct udp_sock), .obj_size = sizeof(struct udp_sock),
.slab_flags = SLAB_DESTROY_BY_RCU,
.h.udp_table = &udp_table, .h.udp_table = &udp_table,
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
.compat_setsockopt = compat_udp_setsockopt, .compat_setsockopt = compat_udp_setsockopt,
......
...@@ -51,6 +51,7 @@ struct proto udplite_prot = { ...@@ -51,6 +51,7 @@ struct proto udplite_prot = {
.unhash = udp_lib_unhash, .unhash = udp_lib_unhash,
.get_port = udp_v4_get_port, .get_port = udp_v4_get_port,
.obj_size = sizeof(struct udp_sock), .obj_size = sizeof(struct udp_sock),
.slab_flags = SLAB_DESTROY_BY_RCU,
.h.udp_table = &udplite_table, .h.udp_table = &udplite_table,
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
.compat_setsockopt = compat_udp_setsockopt, .compat_setsockopt = compat_udp_setsockopt,
......
...@@ -97,24 +97,40 @@ static struct sock *__udp6_lib_lookup(struct net *net, ...@@ -97,24 +97,40 @@ static struct sock *__udp6_lib_lookup(struct net *net,
struct in6_addr *daddr, __be16 dport, struct in6_addr *daddr, __be16 dport,
int dif, struct udp_table *udptable) int dif, struct udp_table *udptable)
{ {
struct sock *sk, *result = NULL; struct sock *sk, *result;
struct hlist_node *node; struct hlist_node *node;
unsigned short hnum = ntohs(dport); unsigned short hnum = ntohs(dport);
unsigned int hash = udp_hashfn(net, hnum); unsigned int hash = udp_hashfn(net, hnum);
struct udp_hslot *hslot = &udptable->hash[hash]; struct udp_hslot *hslot = &udptable->hash[hash];
int score, badness = -1; int score, badness;
spin_lock(&hslot->lock); rcu_read_lock();
sk_for_each(sk, node, &hslot->head) { begin:
result = NULL;
badness = -1;
sk_for_each_rcu(sk, node, &hslot->head) {
/*
* lockless reader, and SLAB_DESTROY_BY_RCU items:
* We must check this item was not moved to another chain
*/
if (udp_hashfn(net, sk->sk_hash) != hash)
goto begin;
score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif); score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif);
if (score > badness) { if (score > badness) {
result = sk; result = sk;
badness = score; badness = score;
} }
} }
if (result) if (result) {
sock_hold(result); if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
spin_unlock(&hslot->lock); result = NULL;
else if (unlikely(compute_score(result, net, hnum, saddr, sport,
daddr, dport, dif) < badness)) {
sock_put(result);
goto begin;
}
}
rcu_read_unlock();
return result; return result;
} }
...@@ -1062,6 +1078,7 @@ struct proto udpv6_prot = { ...@@ -1062,6 +1078,7 @@ struct proto udpv6_prot = {
.sysctl_wmem = &sysctl_udp_wmem_min, .sysctl_wmem = &sysctl_udp_wmem_min,
.sysctl_rmem = &sysctl_udp_rmem_min, .sysctl_rmem = &sysctl_udp_rmem_min,
.obj_size = sizeof(struct udp6_sock), .obj_size = sizeof(struct udp6_sock),
.slab_flags = SLAB_DESTROY_BY_RCU,
.h.udp_table = &udp_table, .h.udp_table = &udp_table,
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
.compat_setsockopt = compat_udpv6_setsockopt, .compat_setsockopt = compat_udpv6_setsockopt,
......
...@@ -49,6 +49,7 @@ struct proto udplitev6_prot = { ...@@ -49,6 +49,7 @@ struct proto udplitev6_prot = {
.unhash = udp_lib_unhash, .unhash = udp_lib_unhash,
.get_port = udp_v6_get_port, .get_port = udp_v6_get_port,
.obj_size = sizeof(struct udp6_sock), .obj_size = sizeof(struct udp6_sock),
.slab_flags = SLAB_DESTROY_BY_RCU,
.h.udp_table = &udplite_table, .h.udp_table = &udplite_table,
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
.compat_setsockopt = compat_udpv6_setsockopt, .compat_setsockopt = compat_udpv6_setsockopt,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment