Skip to content

L
linux
Commit 15f41e2b authored by David S. Miller's avatar David S. Miller
Browse files
Options

Merge branch 'tcp-udp-misc' 

Eric Dumazet says:

====================
net: various udp/tcp changes

First round of patches for linux-4.7

Add a generic facility for sockets to be freed after an RCU grace
period, if they need to.

Then UDP stack is changed to no longer use SLAB_DESTROY_BY_RCU,
in order to speedup rx processing for traffic encapsulated in UDP.
It gives a 17 % speedup for normal UDP reception in stress conditions.

Then TCP listeners are changed to use SOCK_RCU_FREE as well
to avoid touching sk_refcnt in synflood case :
I got up to 30 % performance increase for a mono listener.

Then three patches add SK_MEMINFO_DROPS to sock_diag
and add per socket rx drops accounting to TCP.

Last patch adds rate limiting on ACK sent on behalf of SYN_RECV
to better resist to SYNFLOOD targeting one or few flows.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 43e2dfb2 4ce7e93c
Show whitespace changes
Inline Side-by-side
Showing with 401 additions and 578 deletions
include/linux/udp.h
View file @ 15f41e2b
...@@ -98,11 +98,11 @@ static inline bool udp_get_no_check6_rx(struct sock *sk) ...@@ -98,11 +98,11 @@ static inline bool udp_get_no_check6_rx(struct sock *sk)
return udp_sk(sk)->no_check6_rx; return udp_sk(sk)->no_check6_rx;
} }
#define udp_portaddr_for_each_entry(__sk, node, list) \ #define udp_portaddr_for_each_entry(__sk, list) \
hlist_nulls_for_each_entry(__sk, node, list, __sk_common.skc_portaddr_node) hlist_for_each_entry(__sk, list, __sk_common.skc_portaddr_node)
#define udp_portaddr_for_each_entry_rcu(__sk, node, list) \ #define udp_portaddr_for_each_entry_rcu(__sk, list) \
hlist_nulls_for_each_entry_rcu(__sk, node, list, __sk_common.skc_portaddr_node) hlist_for_each_entry_rcu(__sk, list, __sk_common.skc_portaddr_node)
#define IS_UDPLITE(__sk) (udp_sk(__sk)->pcflag) #define IS_UDPLITE(__sk) (udp_sk(__sk)->pcflag)
......
include/net/inet6_hashtables.h
View file @ 15f41e2b
...@@ -66,13 +66,15 @@ static inline struct sock *__inet6_lookup(struct net *net, ...@@ -66,13 +66,15 @@ static inline struct sock *__inet6_lookup(struct net *net,
const __be16 sport, const __be16 sport,
const struct in6_addr *daddr, const struct in6_addr *daddr,
const u16 hnum, const u16 hnum,
const int dif) const int dif,
bool *refcounted)
{ {
struct sock *sk = __inet6_lookup_established(net, hashinfo, saddr, struct sock *sk = __inet6_lookup_established(net, hashinfo, saddr,
sport, daddr, hnum, dif); sport, daddr, hnum, dif);
*refcounted = true;
if (sk) if (sk)
return sk; return sk;
*refcounted = false;
return inet6_lookup_listener(net, hashinfo, skb, doff, saddr, sport, return inet6_lookup_listener(net, hashinfo, skb, doff, saddr, sport,
daddr, hnum, dif); daddr, hnum, dif);
} }
...@@ -81,17 +83,19 @@ static inline struct sock *__inet6_lookup_skb(struct inet_hashinfo *hashinfo, ...@@ -81,17 +83,19 @@ static inline struct sock *__inet6_lookup_skb(struct inet_hashinfo *hashinfo,
struct sk_buff *skb, int doff, struct sk_buff *skb, int doff,
const __be16 sport, const __be16 sport,
const __be16 dport, const __be16 dport,
int iif) int iif,
bool *refcounted)
{ {
struct sock *sk = skb_steal_sock(skb); struct sock *sk = skb_steal_sock(skb);
*refcounted = true;
if (sk) if (sk)
return sk; return sk;
return __inet6_lookup(dev_net(skb_dst(skb)->dev), hashinfo, skb, return __inet6_lookup(dev_net(skb_dst(skb)->dev), hashinfo, skb,
doff, &ipv6_hdr(skb)->saddr, sport, doff, &ipv6_hdr(skb)->saddr, sport,
&ipv6_hdr(skb)->daddr, ntohs(dport), &ipv6_hdr(skb)->daddr, ntohs(dport),
iif); iif, refcounted);
} }
struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo, struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
......
include/net/inet_hashtables.h
View file @ 15f41e2b
...@@ -100,14 +100,10 @@ struct inet_bind_hashbucket { ...@@ -100,14 +100,10 @@ struct inet_bind_hashbucket {
/* /*
* Sockets can be hashed in established or listening table * Sockets can be hashed in established or listening table
* We must use different 'nulls' end-of-chain value for listening
* hash table, or we might find a socket that was closed and
* reallocated/inserted into established hash table
*/ */
#define LISTENING_NULLS_BASE (1U << 29)
struct inet_listen_hashbucket { struct inet_listen_hashbucket {
spinlock_t lock; spinlock_t lock;
struct hlist_nulls_head head; struct hlist_head head;
}; };
/* This is for listening sockets, thus all sockets which possess wildcards. */ /* This is for listening sockets, thus all sockets which possess wildcards. */
...@@ -280,11 +276,8 @@ static inline struct sock *inet_lookup_listener(struct net *net, ...@@ -280,11 +276,8 @@ static inline struct sock *inet_lookup_listener(struct net *net,
net_eq(sock_net(__sk), (__net))) net_eq(sock_net(__sk), (__net)))
#endif /* 64-bit arch */ #endif /* 64-bit arch */
/* /* Sockets in TCP_CLOSE state are _always_ taken out of the hash, so we need
* Sockets in TCP_CLOSE state are _always_ taken out of the hash, so we need
* not check it for lookups anymore, thanks Alexey. -DaveM * not check it for lookups anymore, thanks Alexey. -DaveM
*
* Local BH must be disabled here.
*/ */
struct sock *__inet_lookup_established(struct net *net, struct sock *__inet_lookup_established(struct net *net,
struct inet_hashinfo *hashinfo, struct inet_hashinfo *hashinfo,
...@@ -307,13 +300,19 @@ static inline struct sock *__inet_lookup(struct net *net, ...@@ -307,13 +300,19 @@ static inline struct sock *__inet_lookup(struct net *net,
struct sk_buff *skb, int doff, struct sk_buff *skb, int doff,
const __be32 saddr, const __be16 sport, const __be32 saddr, const __be16 sport,
const __be32 daddr, const __be16 dport, const __be32 daddr, const __be16 dport,
const int dif) const int dif,
bool *refcounted)
{ {
u16 hnum = ntohs(dport); u16 hnum = ntohs(dport);
struct sock *sk = __inet_lookup_established(net, hashinfo, struct sock *sk;
saddr, sport, daddr, hnum, dif);
return sk ? : __inet_lookup_listener(net, hashinfo, skb, doff, saddr, sk = __inet_lookup_established(net, hashinfo, saddr, sport,
daddr, hnum, dif);
*refcounted = true;
if (sk)
return sk;
*refcounted = false;
return __inet_lookup_listener(net, hashinfo, skb, doff, saddr,
sport, daddr, hnum, dif); sport, daddr, hnum, dif);
} }
...@@ -325,12 +324,13 @@ static inline struct sock *inet_lookup(struct net *net, ...@@ -325,12 +324,13 @@ static inline struct sock *inet_lookup(struct net *net,
const int dif) const int dif)
{ {
struct sock *sk; struct sock *sk;
bool refcounted;
local_bh_disable();
sk = __inet_lookup(net, hashinfo, skb, doff, saddr, sport, daddr, sk = __inet_lookup(net, hashinfo, skb, doff, saddr, sport, daddr,
dport, dif); dport, dif, &refcounted);
local_bh_enable();
if (sk && !refcounted && !atomic_inc_not_zero(&sk->sk_refcnt))
sk = NULL;
return sk; return sk;
} }
...@@ -338,17 +338,20 @@ static inline struct sock *__inet_lookup_skb(struct inet_hashinfo *hashinfo, ...@@ -338,17 +338,20 @@ static inline struct sock *__inet_lookup_skb(struct inet_hashinfo *hashinfo,
struct sk_buff *skb, struct sk_buff *skb,
int doff, int doff,
const __be16 sport, const __be16 sport,
const __be16 dport) const __be16 dport,
bool *refcounted)
{ {
struct sock *sk = skb_steal_sock(skb); struct sock *sk = skb_steal_sock(skb);
const struct iphdr *iph = ip_hdr(skb); const struct iphdr *iph = ip_hdr(skb);
*refcounted = true;
if (sk) if (sk)
return sk; return sk;
else
return __inet_lookup(dev_net(skb_dst(skb)->dev), hashinfo, skb, return __inet_lookup(dev_net(skb_dst(skb)->dev), hashinfo, skb,
doff, iph->saddr, sport, doff, iph->saddr, sport,
iph->daddr, dport, inet_iif(skb)); iph->daddr, dport, inet_iif(skb),
refcounted);
} }
u32 sk_ehashfn(const struct sock *sk); u32 sk_ehashfn(const struct sock *sk);
......
include/net/request_sock.h
View file @ 15f41e2b
...@@ -85,24 +85,23 @@ reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener, ...@@ -85,24 +85,23 @@ reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener,
struct request_sock *req; struct request_sock *req;
req = kmem_cache_alloc(ops->slab, GFP_ATOMIC | __GFP_NOWARN); req = kmem_cache_alloc(ops->slab, GFP_ATOMIC | __GFP_NOWARN);
if (!req)
if (req) { return NULL;
req->rsk_ops = ops; req->rsk_listener = NULL;
if (attach_listener) { if (attach_listener) {
sock_hold(sk_listener); if (unlikely(!atomic_inc_not_zero(&sk_listener->sk_refcnt))) {
kmem_cache_free(ops->slab, req);
return NULL;
}
req->rsk_listener = sk_listener; req->rsk_listener = sk_listener;
} else {
req->rsk_listener = NULL;
} }
req->rsk_ops = ops;
req_to_sk(req)->sk_prot = sk_listener->sk_prot; req_to_sk(req)->sk_prot = sk_listener->sk_prot;
sk_node_init(&req_to_sk(req)->sk_node); sk_node_init(&req_to_sk(req)->sk_node);
sk_tx_queue_clear(req_to_sk(req)); sk_tx_queue_clear(req_to_sk(req));
req->saved_syn = NULL; req->saved_syn = NULL;
/* Following is temporary. It is coupled with debugging
* helpers in reqsk_put() & reqsk_free()
*/
atomic_set(&req->rsk_refcnt, 0); atomic_set(&req->rsk_refcnt, 0);
}
return req; return req;
} }
......
include/net/sock.h
View file @ 15f41e2b
...@@ -178,7 +178,7 @@ struct sock_common { ...@@ -178,7 +178,7 @@ struct sock_common {
int skc_bound_dev_if; int skc_bound_dev_if;
union { union {
struct hlist_node skc_bind_node; struct hlist_node skc_bind_node;
struct hlist_nulls_node skc_portaddr_node; struct hlist_node skc_portaddr_node;
}; };
struct proto *skc_prot; struct proto *skc_prot;
possible_net_t skc_net; possible_net_t skc_net;
...@@ -438,6 +438,7 @@ struct sock { ...@@ -438,6 +438,7 @@ struct sock {
struct sk_buff *skb); struct sk_buff *skb);
void (*sk_destruct)(struct sock *sk); void (*sk_destruct)(struct sock *sk);
struct sock_reuseport __rcu *sk_reuseport_cb; struct sock_reuseport __rcu *sk_reuseport_cb;
struct rcu_head sk_rcu;
}; };
#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data))) #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
...@@ -669,18 +670,18 @@ static inline void sk_add_bind_node(struct sock *sk, ...@@ -669,18 +670,18 @@ static inline void sk_add_bind_node(struct sock *sk,
hlist_for_each_entry(__sk, list, sk_bind_node) hlist_for_each_entry(__sk, list, sk_bind_node)
/** /**
* sk_nulls_for_each_entry_offset - iterate over a list at a given struct offset * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset
* @tpos: the type * to use as a loop cursor. * @tpos: the type * to use as a loop cursor.
* @pos: the &struct hlist_node to use as a loop cursor. * @pos: the &struct hlist_node to use as a loop cursor.
* @head: the head for your list. * @head: the head for your list.
* @offset: offset of hlist_node within the struct. * @offset: offset of hlist_node within the struct.
* *
*/ */
#define sk_nulls_for_each_entry_offset(tpos, pos, head, offset) \ #define sk_for_each_entry_offset_rcu(tpos, pos, head, offset) \
for (pos = (head)->first; \ for (pos = rcu_dereference((head)->first); \
(!is_a_nulls(pos)) && \ pos != NULL && \
({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \ ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \
pos = pos->next) pos = rcu_dereference(pos->next))
static inline struct user_namespace *sk_user_ns(struct sock *sk) static inline struct user_namespace *sk_user_ns(struct sock *sk)
{ {
...@@ -720,6 +721,7 @@ enum sock_flags { ...@@ -720,6 +721,7 @@ enum sock_flags {
*/ */
SOCK_FILTER_LOCKED, /* Filter cannot be changed anymore */ SOCK_FILTER_LOCKED, /* Filter cannot be changed anymore */
SOCK_SELECT_ERR_QUEUE, /* Wake select on error queue */ SOCK_SELECT_ERR_QUEUE, /* Wake select on error queue */
SOCK_RCU_FREE, /* wait rcu grace period in sk_destruct() */
}; };
#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)) #define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
...@@ -2010,6 +2012,13 @@ sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb) ...@@ -2010,6 +2012,13 @@ sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
SOCK_SKB_CB(skb)->dropcount = atomic_read(&sk->sk_drops); SOCK_SKB_CB(skb)->dropcount = atomic_read(&sk->sk_drops);
} }
static inline void sk_drops_add(struct sock *sk, const struct sk_buff *skb)
{
int segs = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
atomic_add(segs, &sk->sk_drops);
}
void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk, void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
struct sk_buff *skb); struct sk_buff *skb);
void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk, void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
......
include/net/tcp.h
View file @ 15f41e2b
...@@ -1836,4 +1836,17 @@ static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) ...@@ -1836,4 +1836,17 @@ static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
tp->data_segs_in += segs_in; tp->data_segs_in += segs_in;
} }
/*
* TCP listen path runs lockless.
* We forced "struct sock" to be const qualified to make sure
* we don't modify one of its field by mistake.
* Here, we increment sk_drops which is an atomic_t, so we can safely
* make sock writable again.
*/
static inline void tcp_listendrop(const struct sock *sk)
{
atomic_inc(&((struct sock *)sk)->sk_drops);
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS);
}
#endif /* _TCP_H */ #endif /* _TCP_H */
include/net/udp.h
View file @ 15f41e2b
...@@ -59,7 +59,7 @@ struct udp_skb_cb { ...@@ -59,7 +59,7 @@ struct udp_skb_cb {
* @lock: spinlock protecting changes to head/count * @lock: spinlock protecting changes to head/count
*/ */
struct udp_hslot { struct udp_hslot {
struct hlist_nulls_head head; struct hlist_head head;
int count; int count;
spinlock_t lock; spinlock_t lock;
} __attribute__((aligned(2 * sizeof(long)))); } __attribute__((aligned(2 * sizeof(long))));
......
include/uapi/linux/sock_diag.h
View file @ 15f41e2b
...@@ -20,6 +20,7 @@ enum { ...@@ -20,6 +20,7 @@ enum {
SK_MEMINFO_WMEM_QUEUED, SK_MEMINFO_WMEM_QUEUED,
SK_MEMINFO_OPTMEM, SK_MEMINFO_OPTMEM,
SK_MEMINFO_BACKLOG, SK_MEMINFO_BACKLOG,
SK_MEMINFO_DROPS,
SK_MEMINFO_VARS, SK_MEMINFO_VARS,
}; };
......
net/core/sock.c
View file @ 15f41e2b
...@@ -1419,8 +1419,12 @@ struct sock *sk_alloc(struct net *net, int family, gfp_t priority, ...@@ -1419,8 +1419,12 @@ struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
} }
EXPORT_SYMBOL(sk_alloc); EXPORT_SYMBOL(sk_alloc);
void sk_destruct(struct sock *sk) /* Sockets having SOCK_RCU_FREE will call this function after one RCU
* grace period. This is the case for UDP sockets and TCP listeners.
*/
static void __sk_destruct(struct rcu_head *head)
{ {
struct sock *sk = container_of(head, struct sock, sk_rcu);
struct sk_filter *filter; struct sk_filter *filter;
if (sk->sk_destruct) if (sk->sk_destruct)
...@@ -1449,6 +1453,14 @@ void sk_destruct(struct sock *sk) ...@@ -1449,6 +1453,14 @@ void sk_destruct(struct sock *sk)
sk_prot_free(sk->sk_prot_creator, sk); sk_prot_free(sk->sk_prot_creator, sk);
} }
void sk_destruct(struct sock *sk)
{
if (sock_flag(sk, SOCK_RCU_FREE))
call_rcu(&sk->sk_rcu, __sk_destruct);
else
__sk_destruct(&sk->sk_rcu);
}
static void __sk_free(struct sock *sk) static void __sk_free(struct sock *sk)
{ {
if (unlikely(sock_diag_has_destroy_listeners(sk) && sk->sk_net_refcnt)) if (unlikely(sock_diag_has_destroy_listeners(sk) && sk->sk_net_refcnt))
...@@ -1513,6 +1525,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) ...@@ -1513,6 +1525,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
newsk->sk_dst_cache = NULL; newsk->sk_dst_cache = NULL;
newsk->sk_wmem_queued = 0; newsk->sk_wmem_queued = 0;
newsk->sk_forward_alloc = 0; newsk->sk_forward_alloc = 0;
atomic_set(&newsk->sk_drops, 0);
newsk->sk_send_head = NULL; newsk->sk_send_head = NULL;
newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
......
net/core/sock_diag.c
View file @ 15f41e2b
...@@ -67,6 +67,7 @@ int sock_diag_put_meminfo(struct sock *sk, struct sk_buff *skb, int attrtype) ...@@ -67,6 +67,7 @@ int sock_diag_put_meminfo(struct sock *sk, struct sk_buff *skb, int attrtype)
mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued; mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued;
mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len; mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len;
mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
return nla_put(skb, attrtype, sizeof(mem), &mem); return nla_put(skb, attrtype, sizeof(mem), &mem);
} }
......
net/dccp/ipv4.c
View file @ 15f41e2b
...@@ -764,6 +764,7 @@ static int dccp_v4_rcv(struct sk_buff *skb) ...@@ -764,6 +764,7 @@ static int dccp_v4_rcv(struct sk_buff *skb)
{ {
const struct dccp_hdr *dh; const struct dccp_hdr *dh;
const struct iphdr *iph; const struct iphdr *iph;
bool refcounted;
struct sock *sk; struct sock *sk;
int min_cov; int min_cov;
...@@ -801,7 +802,7 @@ static int dccp_v4_rcv(struct sk_buff *skb) ...@@ -801,7 +802,7 @@ static int dccp_v4_rcv(struct sk_buff *skb)
lookup: lookup:
sk = __inet_lookup_skb(&dccp_hashinfo, skb, __dccp_hdr_len(dh), sk = __inet_lookup_skb(&dccp_hashinfo, skb, __dccp_hdr_len(dh),
dh->dccph_sport, dh->dccph_dport); dh->dccph_sport, dh->dccph_dport, &refcounted);
if (!sk) { if (!sk) {
dccp_pr_debug("failed to look up flow ID in table and " dccp_pr_debug("failed to look up flow ID in table and "
"get corresponding socket\n"); "get corresponding socket\n");
...@@ -830,6 +831,7 @@ static int dccp_v4_rcv(struct sk_buff *skb) ...@@ -830,6 +831,7 @@ static int dccp_v4_rcv(struct sk_buff *skb)
goto lookup; goto lookup;
} }
sock_hold(sk); sock_hold(sk);
refcounted = true;
nsk = dccp_check_req(sk, skb, req); nsk = dccp_check_req(sk, skb, req);
if (!nsk) { if (!nsk) {
reqsk_put(req); reqsk_put(req);
...@@ -886,6 +888,7 @@ static int dccp_v4_rcv(struct sk_buff *skb) ...@@ -886,6 +888,7 @@ static int dccp_v4_rcv(struct sk_buff *skb)
return 0; return 0;
discard_and_relse: discard_and_relse:
if (refcounted)
sock_put(sk); sock_put(sk);
goto discard_it; goto discard_it;
} }
......
net/dccp/ipv6.c
View file @ 15f41e2b
...@@ -642,6 +642,7 @@ static int dccp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) ...@@ -642,6 +642,7 @@ static int dccp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
static int dccp_v6_rcv(struct sk_buff *skb) static int dccp_v6_rcv(struct sk_buff *skb)
{ {
const struct dccp_hdr *dh; const struct dccp_hdr *dh;
bool refcounted;
struct sock *sk; struct sock *sk;
int min_cov; int min_cov;
...@@ -670,7 +671,7 @@ static int dccp_v6_rcv(struct sk_buff *skb) ...@@ -670,7 +671,7 @@ static int dccp_v6_rcv(struct sk_buff *skb)
lookup: lookup:
sk = __inet6_lookup_skb(&dccp_hashinfo, skb, __dccp_hdr_len(dh), sk = __inet6_lookup_skb(&dccp_hashinfo, skb, __dccp_hdr_len(dh),
dh->dccph_sport, dh->dccph_dport, dh->dccph_sport, dh->dccph_dport,
inet6_iif(skb)); inet6_iif(skb), &refcounted);
if (!sk) { if (!sk) {
dccp_pr_debug("failed to look up flow ID in table and " dccp_pr_debug("failed to look up flow ID in table and "
"get corresponding socket\n"); "get corresponding socket\n");
...@@ -699,6 +700,7 @@ static int dccp_v6_rcv(struct sk_buff *skb) ...@@ -699,6 +700,7 @@ static int dccp_v6_rcv(struct sk_buff *skb)
goto lookup; goto lookup;
} }
sock_hold(sk); sock_hold(sk);
refcounted = true;
nsk = dccp_check_req(sk, skb, req); nsk = dccp_check_req(sk, skb, req);
if (!nsk) { if (!nsk) {
reqsk_put(req); reqsk_put(req);
...@@ -752,6 +754,7 @@ static int dccp_v6_rcv(struct sk_buff *skb) ...@@ -752,6 +754,7 @@ static int dccp_v6_rcv(struct sk_buff *skb)
return 0; return 0;
discard_and_relse: discard_and_relse:
if (refcounted)
sock_put(sk); sock_put(sk);
goto discard_it; goto discard_it;
} }
......
net/ipv4/inet_diag.c
View file @ 15f41e2b
...@@ -356,6 +356,7 @@ struct sock *inet_diag_find_one_icsk(struct net *net, ...@@ -356,6 +356,7 @@ struct sock *inet_diag_find_one_icsk(struct net *net,
{ {
struct sock *sk; struct sock *sk;
rcu_read_lock();
if (req->sdiag_family == AF_INET) if (req->sdiag_family == AF_INET)
sk = inet_lookup(net, hashinfo, NULL, 0, req->id.idiag_dst[0], sk = inet_lookup(net, hashinfo, NULL, 0, req->id.idiag_dst[0],
req->id.idiag_dport, req->id.idiag_src[0], req->id.idiag_dport, req->id.idiag_src[0],
...@@ -376,9 +377,11 @@ struct sock *inet_diag_find_one_icsk(struct net *net, ...@@ -376,9 +377,11 @@ struct sock *inet_diag_find_one_icsk(struct net *net,
req->id.idiag_if); req->id.idiag_if);
} }
#endif #endif
else else {
rcu_read_unlock();
return ERR_PTR(-EINVAL); return ERR_PTR(-EINVAL);
}
rcu_read_unlock();
if (!sk) if (!sk)
return ERR_PTR(-ENOENT); return ERR_PTR(-ENOENT);
...@@ -772,13 +775,12 @@ void inet_diag_dump_icsk(struct inet_hashinfo *hashinfo, struct sk_buff *skb, ...@@ -772,13 +775,12 @@ void inet_diag_dump_icsk(struct inet_hashinfo *hashinfo, struct sk_buff *skb,
for (i = s_i; i < INET_LHTABLE_SIZE; i++) { for (i = s_i; i < INET_LHTABLE_SIZE; i++) {
struct inet_listen_hashbucket *ilb; struct inet_listen_hashbucket *ilb;
struct hlist_nulls_node *node;
struct sock *sk; struct sock *sk;
num = 0; num = 0;
ilb = &hashinfo->listening_hash[i]; ilb = &hashinfo->listening_hash[i];
spin_lock_bh(&ilb->lock); spin_lock_bh(&ilb->lock);
sk_nulls_for_each(sk, node, &ilb->head) { sk_for_each(sk, &ilb->head) {
struct inet_sock *inet = inet_sk(sk); struct inet_sock *inet = inet_sk(sk);
if (!net_eq(sock_net(sk), net)) if (!net_eq(sock_net(sk), net))
......
net/ipv4/inet_hashtables.c
View file @ 15f41e2b
...@@ -198,13 +198,13 @@ static inline int compute_score(struct sock *sk, struct net *net, ...@@ -198,13 +198,13 @@ static inline int compute_score(struct sock *sk, struct net *net,
} }
/* /*
* Don't inline this cruft. Here are some nice properties to exploit here. The * Here are some nice properties to exploit here. The BSD API
* BSD API does not allow a listening sock to specify the remote port nor the * does not allow a listening sock to specify the remote port nor the
* remote address for the connection. So always assume those are both * remote address for the connection. So always assume those are both
* wildcarded during the search since they can never be otherwise. * wildcarded during the search since they can never be otherwise.
*/ */
/* called with rcu_read_lock() : No refcount taken on the socket */
struct sock *__inet_lookup_listener(struct net *net, struct sock *__inet_lookup_listener(struct net *net,
struct inet_hashinfo *hashinfo, struct inet_hashinfo *hashinfo,
struct sk_buff *skb, int doff, struct sk_buff *skb, int doff,
...@@ -212,38 +212,27 @@ struct sock *__inet_lookup_listener(struct net *net, ...@@ -212,38 +212,27 @@ struct sock *__inet_lookup_listener(struct net *net,
const __be32 daddr, const unsigned short hnum, const __be32 daddr, const unsigned short hnum,
const int dif) const int dif)
{ {
struct sock *sk, *result;
struct hlist_nulls_node *node;
unsigned int hash = inet_lhashfn(net, hnum); unsigned int hash = inet_lhashfn(net, hnum);
struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
int score, hiscore, matches = 0, reuseport = 0; int score, hiscore = 0, matches = 0, reuseport = 0;
bool select_ok = true; struct sock *sk, *result = NULL;
u32 phash = 0; u32 phash = 0;
rcu_read_lock(); sk_for_each_rcu(sk, &ilb->head) {
begin:
result = NULL;
hiscore = 0;
sk_nulls_for_each_rcu(sk, node, &ilb->head) {
score = compute_score(sk, net, hnum, daddr, dif); score = compute_score(sk, net, hnum, daddr, dif);
if (score > hiscore) { if (score > hiscore) {
result = sk;
hiscore = score;
reuseport = sk->sk_reuseport; reuseport = sk->sk_reuseport;
if (reuseport) { if (reuseport) {
phash = inet_ehashfn(net, daddr, hnum, phash = inet_ehashfn(net, daddr, hnum,
saddr, sport); saddr, sport);
if (select_ok) { result = reuseport_select_sock(sk, phash,
struct sock *sk2;
sk2 = reuseport_select_sock(sk, phash,
skb, doff); skb, doff);
if (sk2) { if (result)
result = sk2; return result;
goto found;
}
}
matches = 1; matches = 1;
} }
result = sk;
hiscore = score;
} else if (score == hiscore && reuseport) { } else if (score == hiscore && reuseport) {
matches++; matches++;
if (reciprocal_scale(phash, matches) == 0) if (reciprocal_scale(phash, matches) == 0)
...@@ -251,25 +240,6 @@ struct sock *__inet_lookup_listener(struct net *net, ...@@ -251,25 +240,6 @@ struct sock *__inet_lookup_listener(struct net *net,
phash = next_pseudo_random32(phash); phash = next_pseudo_random32(phash);
} }
} }
/*
* if the nulls value we got at the end of this lookup is
* not the expected one, we must restart lookup.
* We probably met an item that was moved to another chain.
*/
if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
goto begin;
if (result) {
found:
if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
result = NULL;
else if (unlikely(compute_score(result, net, hnum, daddr,
dif) < hiscore)) {
sock_put(result);
select_ok = false;
goto begin;
}
}
rcu_read_unlock();
return result; return result;
} }
EXPORT_SYMBOL_GPL(__inet_lookup_listener); EXPORT_SYMBOL_GPL(__inet_lookup_listener);
...@@ -312,7 +282,6 @@ struct sock *__inet_lookup_established(struct net *net, ...@@ -312,7 +282,6 @@ struct sock *__inet_lookup_established(struct net *net,
unsigned int slot = hash & hashinfo->ehash_mask; unsigned int slot = hash & hashinfo->ehash_mask;
struct inet_ehash_bucket *head = &hashinfo->ehash[slot]; struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
rcu_read_lock();
begin: begin:
sk_nulls_for_each_rcu(sk, node, &head->chain) { sk_nulls_for_each_rcu(sk, node, &head->chain) {
if (sk->sk_hash != hash) if (sk->sk_hash != hash)
...@@ -339,7 +308,6 @@ struct sock *__inet_lookup_established(struct net *net, ...@@ -339,7 +308,6 @@ struct sock *__inet_lookup_established(struct net *net,
out: out:
sk = NULL; sk = NULL;
found: found:
rcu_read_unlock();
return sk; return sk;
} }
EXPORT_SYMBOL_GPL(__inet_lookup_established); EXPORT_SYMBOL_GPL(__inet_lookup_established);
...@@ -512,7 +480,8 @@ int __inet_hash(struct sock *sk, struct sock *osk, ...@@ -512,7 +480,8 @@ int __inet_hash(struct sock *sk, struct sock *osk,
if (err) if (err)
goto unlock; goto unlock;
} }
__sk_nulls_add_node_rcu(sk, &ilb->head); hlist_add_head_rcu(&sk->sk_node, &ilb->head);
sock_set_flag(sk, SOCK_RCU_FREE);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
unlock: unlock:
spin_unlock(&ilb->lock); spin_unlock(&ilb->lock);
...@@ -539,19 +508,24 @@ void inet_unhash(struct sock *sk) ...@@ -539,19 +508,24 @@ void inet_unhash(struct sock *sk)
{ {
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
spinlock_t *lock; spinlock_t *lock;
bool listener = false;
int done; int done;
if (sk_unhashed(sk)) if (sk_unhashed(sk))
return; return;
if (sk->sk_state == TCP_LISTEN) if (sk->sk_state == TCP_LISTEN) {
lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock; lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock;
else listener = true;
} else {
lock = inet_ehash_lockp(hashinfo, sk->sk_hash); lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
}
spin_lock_bh(lock); spin_lock_bh(lock);
if (rcu_access_pointer(sk->sk_reuseport_cb)) if (rcu_access_pointer(sk->sk_reuseport_cb))
reuseport_detach_sock(sk); reuseport_detach_sock(sk);
if (listener)
done = __sk_del_node_init(sk);
else
done = __sk_nulls_del_node_init_rcu(sk); done = __sk_nulls_del_node_init_rcu(sk);
if (done) if (done)
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
...@@ -688,8 +662,7 @@ void inet_hashinfo_init(struct inet_hashinfo *h) ...@@ -688,8 +662,7 @@ void inet_hashinfo_init(struct inet_hashinfo *h)
for (i = 0; i < INET_LHTABLE_SIZE; i++) { for (i = 0; i < INET_LHTABLE_SIZE; i++) {
spin_lock_init(&h->listening_hash[i].lock); spin_lock_init(&h->listening_hash[i].lock);
INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].head, INIT_HLIST_HEAD(&h->listening_hash[i].head);
i + LISTENING_NULLS_BASE);
} }
} }
EXPORT_SYMBOL_GPL(inet_hashinfo_init); EXPORT_SYMBOL_GPL(inet_hashinfo_init);
......
net/ipv4/tcp_input.c
View file @ 15f41e2b
...@@ -4307,6 +4307,12 @@ static bool tcp_try_coalesce(struct sock *sk, ...@@ -4307,6 +4307,12 @@ static bool tcp_try_coalesce(struct sock *sk,
return true; return true;
} }
static void tcp_drop(struct sock *sk, struct sk_buff *skb)
{
sk_drops_add(sk, skb);
__kfree_skb(skb);
}
/* This one checks to see if we can put data from the /* This one checks to see if we can put data from the
* out_of_order queue into the receive_queue. * out_of_order queue into the receive_queue.
*/ */
...@@ -4331,7 +4337,7 @@ static void tcp_ofo_queue(struct sock *sk) ...@@ -4331,7 +4337,7 @@ static void tcp_ofo_queue(struct sock *sk)
__skb_unlink(skb, &tp->out_of_order_queue); __skb_unlink(skb, &tp->out_of_order_queue);
if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) { if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) {
SOCK_DEBUG(sk, "ofo packet was already received\n"); SOCK_DEBUG(sk, "ofo packet was already received\n");
__kfree_skb(skb); tcp_drop(sk, skb);
continue; continue;
} }
SOCK_DEBUG(sk, "ofo requeuing : rcv_next %X seq %X - %X\n", SOCK_DEBUG(sk, "ofo requeuing : rcv_next %X seq %X - %X\n",
...@@ -4383,7 +4389,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) ...@@ -4383,7 +4389,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) { if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) {
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFODROP); NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFODROP);
__kfree_skb(skb); tcp_drop(sk, skb);
return; return;
} }
...@@ -4447,7 +4453,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) ...@@ -4447,7 +4453,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) { if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
/* All the bits are present. Drop. */ /* All the bits are present. Drop. */
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE); NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE);
__kfree_skb(skb); tcp_drop(sk, skb);
skb = NULL; skb = NULL;
tcp_dsack_set(sk, seq, end_seq); tcp_dsack_set(sk, seq, end_seq);
goto add_sack; goto add_sack;
...@@ -4486,7 +4492,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) ...@@ -4486,7 +4492,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq, tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
TCP_SKB_CB(skb1)->end_seq); TCP_SKB_CB(skb1)->end_seq);
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE); NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE);
__kfree_skb(skb1); tcp_drop(sk, skb1);
} }
add_sack: add_sack:
...@@ -4569,12 +4575,13 @@ int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size) ...@@ -4569,12 +4575,13 @@ int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size)
static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
{ {
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
int eaten = -1;
bool fragstolen = false; bool fragstolen = false;
int eaten = -1;
if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
goto drop; __kfree_skb(skb);
return;
}
skb_dst_drop(skb); skb_dst_drop(skb);
__skb_pull(skb, tcp_hdr(skb)->doff * 4); __skb_pull(skb, tcp_hdr(skb)->doff * 4);
...@@ -4656,7 +4663,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) ...@@ -4656,7 +4663,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
tcp_enter_quickack_mode(sk); tcp_enter_quickack_mode(sk);
inet_csk_schedule_ack(sk); inet_csk_schedule_ack(sk);
drop: drop:
__kfree_skb(skb); tcp_drop(sk, skb);
return; return;
} }
...@@ -5233,7 +5240,7 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, ...@@ -5233,7 +5240,7 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
return true; return true;
discard: discard:
__kfree_skb(skb); tcp_drop(sk, skb);
return false; return false;
} }
...@@ -5451,7 +5458,7 @@ void tcp_rcv_established(struct sock *sk, struct sk_buff *skb, ...@@ -5451,7 +5458,7 @@ void tcp_rcv_established(struct sock *sk, struct sk_buff *skb,
TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS); TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS);
discard: discard:
__kfree_skb(skb); tcp_drop(sk, skb);
} }
EXPORT_SYMBOL(tcp_rcv_established); EXPORT_SYMBOL(tcp_rcv_established);
...@@ -5682,7 +5689,7 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, ...@@ -5682,7 +5689,7 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
TCP_DELACK_MAX, TCP_RTO_MAX); TCP_DELACK_MAX, TCP_RTO_MAX);
discard: discard:
__kfree_skb(skb); tcp_drop(sk, skb);
return 0; return 0;
} else { } else {
tcp_send_ack(sk); tcp_send_ack(sk);
...@@ -6043,7 +6050,7 @@ int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) ...@@ -6043,7 +6050,7 @@ int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb)
if (!queued) { if (!queued) {
discard: discard:
__kfree_skb(skb); tcp_drop(sk, skb);
} }
return 0; return 0;
} }
...@@ -6332,8 +6339,10 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, ...@@ -6332,8 +6339,10 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops,
inet_csk_reqsk_queue_hash_add(sk, req, TCP_TIMEOUT_INIT); inet_csk_reqsk_queue_hash_add(sk, req, TCP_TIMEOUT_INIT);
af_ops->send_synack(sk, dst, &fl, req, af_ops->send_synack(sk, dst, &fl, req,
&foc, !want_cookie); &foc, !want_cookie);
if (want_cookie) if (want_cookie) {
goto drop_and_free; reqsk_free(req);
return 0;
}
} }
reqsk_put(req); reqsk_put(req);
return 0; return 0;
...@@ -6343,7 +6352,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, ...@@ -6343,7 +6352,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops,
drop_and_free: drop_and_free:
reqsk_free(req); reqsk_free(req);
drop: drop:
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS); tcp_listendrop(sk);
return 0; return 0;
} }
EXPORT_SYMBOL(tcp_conn_request); EXPORT_SYMBOL(tcp_conn_request);
net/ipv4/tcp_ipv4.c
View file @ 15f41e2b
...@@ -329,7 +329,7 @@ void tcp_req_err(struct sock *sk, u32 seq, bool abort) ...@@ -329,7 +329,7 @@ void tcp_req_err(struct sock *sk, u32 seq, bool abort)
* errors returned from accept(). * errors returned from accept().
*/ */
inet_csk_reqsk_queue_drop(req->rsk_listener, req); inet_csk_reqsk_queue_drop(req->rsk_listener, req);
NET_INC_STATS_BH(net, LINUX_MIB_LISTENDROPS); tcp_listendrop(req->rsk_listener);
} }
reqsk_put(req); reqsk_put(req);
} }
...@@ -628,6 +628,7 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb) ...@@ -628,6 +628,7 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev); net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
#ifdef CONFIG_TCP_MD5SIG #ifdef CONFIG_TCP_MD5SIG
rcu_read_lock();
hash_location = tcp_parse_md5sig_option(th); hash_location = tcp_parse_md5sig_option(th);
if (sk && sk_fullsock(sk)) { if (sk && sk_fullsock(sk)) {
key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *) key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)
...@@ -646,16 +647,18 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb) ...@@ -646,16 +647,18 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
ntohs(th->source), inet_iif(skb)); ntohs(th->source), inet_iif(skb));
/* don't send rst if it can't find key */ /* don't send rst if it can't find key */
if (!sk1) if (!sk1)
return; goto out;
rcu_read_lock();
key = tcp_md5_do_lookup(sk1, (union tcp_md5_addr *) key = tcp_md5_do_lookup(sk1, (union tcp_md5_addr *)
&ip_hdr(skb)->saddr, AF_INET); &ip_hdr(skb)->saddr, AF_INET);
if (!key) if (!key)
goto release_sk1; goto out;
genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb); genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
if (genhash || memcmp(hash_location, newhash, 16) != 0) if (genhash || memcmp(hash_location, newhash, 16) != 0)
goto release_sk1; goto out;
} }
if (key) { if (key) {
...@@ -698,11 +701,8 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb) ...@@ -698,11 +701,8 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
TCP_INC_STATS_BH(net, TCP_MIB_OUTRSTS); TCP_INC_STATS_BH(net, TCP_MIB_OUTRSTS);
#ifdef CONFIG_TCP_MD5SIG #ifdef CONFIG_TCP_MD5SIG
release_sk1: out:
if (sk1) {
rcu_read_unlock(); rcu_read_unlock();
sock_put(sk1);
}
#endif #endif
} }
...@@ -1246,7 +1246,7 @@ int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb) ...@@ -1246,7 +1246,7 @@ int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
&tcp_request_sock_ipv4_ops, sk, skb); &tcp_request_sock_ipv4_ops, sk, skb);
drop: drop:
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS); tcp_listendrop(sk);
return 0; return 0;
} }
EXPORT_SYMBOL(tcp_v4_conn_request); EXPORT_SYMBOL(tcp_v4_conn_request);
...@@ -1348,7 +1348,7 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, ...@@ -1348,7 +1348,7 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
exit_nonewsk: exit_nonewsk:
dst_release(dst); dst_release(dst);
exit: exit:
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS); tcp_listendrop(sk);
return NULL; return NULL;
put_and_exit: put_and_exit:
inet_csk_prepare_forced_close(newsk); inet_csk_prepare_forced_close(newsk);
...@@ -1538,11 +1538,12 @@ EXPORT_SYMBOL(tcp_prequeue); ...@@ -1538,11 +1538,12 @@ EXPORT_SYMBOL(tcp_prequeue);
int tcp_v4_rcv(struct sk_buff *skb) int tcp_v4_rcv(struct sk_buff *skb)
{ {
struct net *net = dev_net(skb->dev);
const struct iphdr *iph; const struct iphdr *iph;
const struct tcphdr *th; const struct tcphdr *th;
bool refcounted;
struct sock *sk; struct sock *sk;
int ret; int ret;
struct net *net = dev_net(skb->dev);
if (skb->pkt_type != PACKET_HOST) if (skb->pkt_type != PACKET_HOST)
goto discard_it; goto discard_it;
...@@ -1588,7 +1589,7 @@ int tcp_v4_rcv(struct sk_buff *skb) ...@@ -1588,7 +1589,7 @@ int tcp_v4_rcv(struct sk_buff *skb)
lookup: lookup:
sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source, sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
th->dest); th->dest, &refcounted);
if (!sk) if (!sk)
goto no_tcp_socket; goto no_tcp_socket;
...@@ -1609,7 +1610,11 @@ int tcp_v4_rcv(struct sk_buff *skb) ...@@ -1609,7 +1610,11 @@ int tcp_v4_rcv(struct sk_buff *skb)
inet_csk_reqsk_queue_drop_and_put(sk, req); inet_csk_reqsk_queue_drop_and_put(sk, req);
goto lookup; goto lookup;
} }
/* We own a reference on the listener, increase it again
* as we might lose it too soon.
*/
sock_hold(sk); sock_hold(sk);
refcounted = true;
nsk = tcp_check_req(sk, skb, req, false); nsk = tcp_check_req(sk, skb, req, false);
if (!nsk) { if (!nsk) {
reqsk_put(req); reqsk_put(req);
...@@ -1665,6 +1670,7 @@ int tcp_v4_rcv(struct sk_buff *skb) ...@@ -1665,6 +1670,7 @@ int tcp_v4_rcv(struct sk_buff *skb)
bh_unlock_sock(sk); bh_unlock_sock(sk);
put_and_return: put_and_return:
if (refcounted)
sock_put(sk); sock_put(sk);
return ret; return ret;
...@@ -1688,6 +1694,8 @@ int tcp_v4_rcv(struct sk_buff *skb) ...@@ -1688,6 +1694,8 @@ int tcp_v4_rcv(struct sk_buff *skb)
return 0; return 0;
discard_and_relse: discard_and_relse:
sk_drops_add(sk, skb);
if (refcounted)
sock_put(sk); sock_put(sk);
goto discard_it; goto discard_it;
...@@ -1712,6 +1720,7 @@ int tcp_v4_rcv(struct sk_buff *skb) ...@@ -1712,6 +1720,7 @@ int tcp_v4_rcv(struct sk_buff *skb)
if (sk2) { if (sk2) {
inet_twsk_deschedule_put(inet_twsk(sk)); inet_twsk_deschedule_put(inet_twsk(sk));
sk = sk2; sk = sk2;
refcounted = false;
goto process; goto process;
} }
/* Fall through to ACK */ /* Fall through to ACK */
...@@ -1845,17 +1854,17 @@ EXPORT_SYMBOL(tcp_v4_destroy_sock); ...@@ -1845,17 +1854,17 @@ EXPORT_SYMBOL(tcp_v4_destroy_sock);
*/ */
static void *listening_get_next(struct seq_file *seq, void *cur) static void *listening_get_next(struct seq_file *seq, void *cur)
{ {
struct inet_connection_sock *icsk;
struct hlist_nulls_node *node;
struct sock *sk = cur;
struct inet_listen_hashbucket *ilb;
struct tcp_iter_state *st = seq->private; struct tcp_iter_state *st = seq->private;
struct net *net = seq_file_net(seq); struct net *net = seq_file_net(seq);
struct inet_listen_hashbucket *ilb;
struct inet_connection_sock *icsk;
struct sock *sk = cur;
if (!sk) { if (!sk) {
get_head:
ilb = &tcp_hashinfo.listening_hash[st->bucket]; ilb = &tcp_hashinfo.listening_hash[st->bucket];
spin_lock_bh(&ilb->lock); spin_lock_bh(&ilb->lock);
sk = sk_nulls_head(&ilb->head); sk = sk_head(&ilb->head);
st->offset = 0; st->offset = 0;
goto get_sk; goto get_sk;
} }
...@@ -1863,28 +1872,20 @@ static void *listening_get_next(struct seq_file *seq, void *cur) ...@@ -1863,28 +1872,20 @@ static void *listening_get_next(struct seq_file *seq, void *cur)
++st->num; ++st->num;
++st->offset; ++st->offset;
sk = sk_nulls_next(sk); sk = sk_next(sk);
get_sk: get_sk:
sk_nulls_for_each_from(sk, node) { sk_for_each_from(sk) {
if (!net_eq(sock_net(sk), net)) if (!net_eq(sock_net(sk), net))
continue; continue;
if (sk->sk_family == st->family) { if (sk->sk_family == st->family)
cur = sk; return sk;
goto out;
}
icsk = inet_csk(sk); icsk = inet_csk(sk);
} }
spin_unlock_bh(&ilb->lock); spin_unlock_bh(&ilb->lock);
st->offset = 0; st->offset = 0;
if (++st->bucket < INET_LHTABLE_SIZE) { if (++st->bucket < INET_LHTABLE_SIZE)
ilb = &tcp_hashinfo.listening_hash[st->bucket]; goto get_head;
spin_lock_bh(&ilb->lock); return NULL;
sk = sk_nulls_head(&ilb->head);
goto get_sk;
}
cur = NULL;
out:
return cur;
} }
static void *listening_get_idx(struct seq_file *seq, loff_t *pos) static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
...@@ -2383,6 +2384,7 @@ static int __net_init tcp_sk_init(struct net *net) ...@@ -2383,6 +2384,7 @@ static int __net_init tcp_sk_init(struct net *net)
IPPROTO_TCP, net); IPPROTO_TCP, net);
if (res) if (res)
goto fail; goto fail;
sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
*per_cpu_ptr(net->ipv4.tcp_sk, cpu) = sk; *per_cpu_ptr(net->ipv4.tcp_sk, cpu) = sk;
} }
......
net/ipv4/tcp_minisocks.c
View file @ 15f41e2b
...@@ -704,7 +704,10 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, ...@@ -704,7 +704,10 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
if (paws_reject || !tcp_in_window(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq, if (paws_reject || !tcp_in_window(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq,
tcp_rsk(req)->rcv_nxt, tcp_rsk(req)->rcv_nxt + req->rsk_rcv_wnd)) { tcp_rsk(req)->rcv_nxt, tcp_rsk(req)->rcv_nxt + req->rsk_rcv_wnd)) {
/* Out of window: send ACK and drop. */ /* Out of window: send ACK and drop. */
if (!(flg & TCP_FLAG_RST)) if (!(flg & TCP_FLAG_RST) &&
!tcp_oow_rate_limited(sock_net(sk), skb,
LINUX_MIB_TCPACKSKIPPEDSYNRECV,
&tcp_rsk(req)->last_oow_ack_time))
req->rsk_ops->send_ack(sk, skb, req); req->rsk_ops->send_ack(sk, skb, req);
if (paws_reject) if (paws_reject)
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED); NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED);
......
net/ipv4/udp.c
View file @ 15f41e2b
...@@ -143,10 +143,9 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num, ...@@ -143,10 +143,9 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num,
unsigned int log) unsigned int log)
{ {
struct sock *sk2; struct sock *sk2;
struct hlist_nulls_node *node;
kuid_t uid = sock_i_uid(sk); kuid_t uid = sock_i_uid(sk);
sk_nulls_for_each(sk2, node, &hslot->head) { sk_for_each(sk2, &hslot->head) {
if (net_eq(sock_net(sk2), net) && if (net_eq(sock_net(sk2), net) &&
sk2 != sk && sk2 != sk &&
(bitmap || udp_sk(sk2)->udp_port_hash == num) && (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
...@@ -177,12 +176,11 @@ static int udp_lib_lport_inuse2(struct net *net, __u16 num, ...@@ -177,12 +176,11 @@ static int udp_lib_lport_inuse2(struct net *net, __u16 num,
bool match_wildcard)) bool match_wildcard))
{ {
struct sock *sk2; struct sock *sk2;
struct hlist_nulls_node *node;
kuid_t uid = sock_i_uid(sk); kuid_t uid = sock_i_uid(sk);
int res = 0; int res = 0;
spin_lock(&hslot2->lock); spin_lock(&hslot2->lock);
udp_portaddr_for_each_entry(sk2, node, &hslot2->head) { udp_portaddr_for_each_entry(sk2, &hslot2->head) {
if (net_eq(sock_net(sk2), net) && if (net_eq(sock_net(sk2), net) &&
sk2 != sk && sk2 != sk &&
(udp_sk(sk2)->udp_port_hash == num) && (udp_sk(sk2)->udp_port_hash == num) &&
...@@ -207,11 +205,10 @@ static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot, ...@@ -207,11 +205,10 @@ static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot,
bool match_wildcard)) bool match_wildcard))
{ {
struct net *net = sock_net(sk); struct net *net = sock_net(sk);
struct hlist_nulls_node *node;
kuid_t uid = sock_i_uid(sk); kuid_t uid = sock_i_uid(sk);
struct sock *sk2; struct sock *sk2;
sk_nulls_for_each(sk2, node, &hslot->head) { sk_for_each(sk2, &hslot->head) {
if (net_eq(sock_net(sk2), net) && if (net_eq(sock_net(sk2), net) &&
sk2 != sk && sk2 != sk &&
sk2->sk_family == sk->sk_family && sk2->sk_family == sk->sk_family &&
...@@ -333,17 +330,18 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, ...@@ -333,17 +330,18 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum,
goto fail_unlock; goto fail_unlock;
} }
sk_nulls_add_node_rcu(sk, &hslot->head); sk_add_node_rcu(sk, &hslot->head);
hslot->count++; hslot->count++;
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
spin_lock(&hslot2->lock); spin_lock(&hslot2->lock);
hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
&hslot2->head); &hslot2->head);
hslot2->count++; hslot2->count++;
spin_unlock(&hslot2->lock); spin_unlock(&hslot2->lock);
} }
sock_set_flag(sk, SOCK_RCU_FREE);
error = 0; error = 0;
fail_unlock: fail_unlock:
spin_unlock_bh(&hslot->lock); spin_unlock_bh(&hslot->lock);
...@@ -497,37 +495,27 @@ static struct sock *udp4_lib_lookup2(struct net *net, ...@@ -497,37 +495,27 @@ static struct sock *udp4_lib_lookup2(struct net *net,
struct sk_buff *skb) struct sk_buff *skb)
{ {
struct sock *sk, *result; struct sock *sk, *result;
struct hlist_nulls_node *node;
int score, badness, matches = 0, reuseport = 0; int score, badness, matches = 0, reuseport = 0;
bool select_ok = true;
u32 hash = 0; u32 hash = 0;
begin:
result = NULL; result = NULL;
badness = 0; badness = 0;
udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) { udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
score = compute_score2(sk, net, saddr, sport, score = compute_score2(sk, net, saddr, sport,
daddr, hnum, dif); daddr, hnum, dif);
if (score > badness) { if (score > badness) {
result = sk;
badness = score;
reuseport = sk->sk_reuseport; reuseport = sk->sk_reuseport;
if (reuseport) { if (reuseport) {
hash = udp_ehashfn(net, daddr, hnum, hash = udp_ehashfn(net, daddr, hnum,
saddr, sport); saddr, sport);
if (select_ok) { result = reuseport_select_sock(sk, hash, skb,
struct sock *sk2;
sk2 = reuseport_select_sock(sk, hash, skb,
sizeof(struct udphdr)); sizeof(struct udphdr));
if (sk2) { if (result)
result = sk2; return result;
select_ok = false;
goto found;
}
}
matches = 1; matches = 1;
} }
badness = score;
result = sk;
} else if (score == badness && reuseport) { } else if (score == badness && reuseport) {
matches++; matches++;
if (reciprocal_scale(hash, matches) == 0) if (reciprocal_scale(hash, matches) == 0)
...@@ -535,23 +523,6 @@ static struct sock *udp4_lib_lookup2(struct net *net, ...@@ -535,23 +523,6 @@ static struct sock *udp4_lib_lookup2(struct net *net,
hash = next_pseudo_random32(hash); hash = next_pseudo_random32(hash);
} }
} }
/*
* if the nulls value we got at the end of this lookup is
* not the expected one, we must restart lookup.
* We probably met an item that was moved to another chain.
*/
if (get_nulls_value(node) != slot2)
goto begin;
if (result) {
found:
if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
result = NULL;
else if (unlikely(compute_score2(result, net, saddr, sport,
daddr, hnum, dif) < badness)) {
sock_put(result);
goto begin;
}
}
return result; return result;
} }
...@@ -563,15 +534,12 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, ...@@ -563,15 +534,12 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
int dif, struct udp_table *udptable, struct sk_buff *skb) int dif, struct udp_table *udptable, struct sk_buff *skb)
{ {
struct sock *sk, *result; struct sock *sk, *result;
struct hlist_nulls_node *node;
unsigned short hnum = ntohs(dport); unsigned short hnum = ntohs(dport);
unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask); unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
struct udp_hslot *hslot2, *hslot = &udptable->hash[slot]; struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
int score, badness, matches = 0, reuseport = 0; int score, badness, matches = 0, reuseport = 0;
bool select_ok = true;
u32 hash = 0; u32 hash = 0;
rcu_read_lock();
if (hslot->count > 10) { if (hslot->count > 10) {
hash2 = udp4_portaddr_hash(net, daddr, hnum); hash2 = udp4_portaddr_hash(net, daddr, hnum);
slot2 = hash2 & udptable->mask; slot2 = hash2 & udptable->mask;
...@@ -593,35 +561,27 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, ...@@ -593,35 +561,27 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
htonl(INADDR_ANY), hnum, dif, htonl(INADDR_ANY), hnum, dif,
hslot2, slot2, skb); hslot2, slot2, skb);
} }
rcu_read_unlock();
return result; return result;
} }
begin: begin:
result = NULL; result = NULL;
badness = 0; badness = 0;
sk_nulls_for_each_rcu(sk, node, &hslot->head) { sk_for_each_rcu(sk, &hslot->head) {
score = compute_score(sk, net, saddr, hnum, sport, score = compute_score(sk, net, saddr, hnum, sport,
daddr, dport, dif); daddr, dport, dif);
if (score > badness) { if (score > badness) {
result = sk;
badness = score;
reuseport = sk->sk_reuseport; reuseport = sk->sk_reuseport;
if (reuseport) { if (reuseport) {
hash = udp_ehashfn(net, daddr, hnum, hash = udp_ehashfn(net, daddr, hnum,
saddr, sport); saddr, sport);
if (select_ok) { result = reuseport_select_sock(sk, hash, skb,
struct sock *sk2;
sk2 = reuseport_select_sock(sk, hash, skb,
sizeof(struct udphdr)); sizeof(struct udphdr));
if (sk2) { if (result)
result = sk2; return result;
select_ok = false;
goto found;
}
}
matches = 1; matches = 1;
} }
result = sk;
badness = score;
} else if (score == badness && reuseport) { } else if (score == badness && reuseport) {
matches++; matches++;
if (reciprocal_scale(hash, matches) == 0) if (reciprocal_scale(hash, matches) == 0)
...@@ -629,25 +589,6 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, ...@@ -629,25 +589,6 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
hash = next_pseudo_random32(hash); hash = next_pseudo_random32(hash);
} }
} }
/*
* if the nulls value we got at the end of this lookup is
* not the expected one, we must restart lookup.
* We probably met an item that was moved to another chain.
*/
if (get_nulls_value(node) != slot)
goto begin;
if (result) {
found:
if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
result = NULL;
else if (unlikely(compute_score(result, net, saddr, hnum, sport,
daddr, dport, dif) < badness)) {
sock_put(result);
goto begin;
}
}
rcu_read_unlock();
return result; return result;
} }
EXPORT_SYMBOL_GPL(__udp4_lib_lookup); EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
...@@ -663,13 +604,24 @@ static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb, ...@@ -663,13 +604,24 @@ static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
udptable, skb); udptable, skb);
} }
/* Must be called under rcu_read_lock().
* Does increment socket refcount.
*/
#if IS_ENABLED(CONFIG_NETFILTER_XT_MATCH_SOCKET) || \
IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TPROXY)
struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport, struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
__be32 daddr, __be16 dport, int dif) __be32 daddr, __be16 dport, int dif)
{ {
return __udp4_lib_lookup(net, saddr, sport, daddr, dport, dif, struct sock *sk;
&udp_table, NULL);
sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport,
dif, &udp_table, NULL);
if (sk && !atomic_inc_not_zero(&sk->sk_refcnt))
sk = NULL;
return sk;
} }
EXPORT_SYMBOL_GPL(udp4_lib_lookup); EXPORT_SYMBOL_GPL(udp4_lib_lookup);
#endif
static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk, static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
__be16 loc_port, __be32 loc_addr, __be16 loc_port, __be32 loc_addr,
...@@ -771,7 +723,7 @@ void __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable) ...@@ -771,7 +723,7 @@ void __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
sk->sk_err = err; sk->sk_err = err;
sk->sk_error_report(sk); sk->sk_error_report(sk);
out: out:
sock_put(sk); return;
} }
void udp_err(struct sk_buff *skb, u32 info) void udp_err(struct sk_buff *skb, u32 info)
...@@ -1474,13 +1426,13 @@ void udp_lib_unhash(struct sock *sk) ...@@ -1474,13 +1426,13 @@ void udp_lib_unhash(struct sock *sk)
spin_lock_bh(&hslot->lock); spin_lock_bh(&hslot->lock);
if (rcu_access_pointer(sk->sk_reuseport_cb)) if (rcu_access_pointer(sk->sk_reuseport_cb))
reuseport_detach_sock(sk); reuseport_detach_sock(sk);
if (sk_nulls_del_node_init_rcu(sk)) { if (sk_del_node_init_rcu(sk)) {
hslot->count--; hslot->count--;
inet_sk(sk)->inet_num = 0; inet_sk(sk)->inet_num = 0;
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
spin_lock(&hslot2->lock); spin_lock(&hslot2->lock);
hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
hslot2->count--; hslot2->count--;
spin_unlock(&hslot2->lock); spin_unlock(&hslot2->lock);
} }
...@@ -1513,12 +1465,12 @@ void udp_lib_rehash(struct sock *sk, u16 newhash) ...@@ -1513,12 +1465,12 @@ void udp_lib_rehash(struct sock *sk, u16 newhash)
if (hslot2 != nhslot2) { if (hslot2 != nhslot2) {
spin_lock(&hslot2->lock); spin_lock(&hslot2->lock);
hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
hslot2->count--; hslot2->count--;
spin_unlock(&hslot2->lock); spin_unlock(&hslot2->lock);
spin_lock(&nhslot2->lock); spin_lock(&nhslot2->lock);
hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
&nhslot2->head); &nhslot2->head);
nhslot2->count++; nhslot2->count++;
spin_unlock(&nhslot2->lock); spin_unlock(&nhslot2->lock);
...@@ -1697,35 +1649,6 @@ int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) ...@@ -1697,35 +1649,6 @@ int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
return -1; return -1;
} }
static void flush_stack(struct sock **stack, unsigned int count,
struct sk_buff *skb, unsigned int final)
{
unsigned int i;
struct sk_buff *skb1 = NULL;
struct sock *sk;
for (i = 0; i < count; i++) {
sk = stack[i];
if (likely(!skb1))
skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
if (!skb1) {
atomic_inc(&sk->sk_drops);
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
IS_UDPLITE(sk));
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
IS_UDPLITE(sk));
}
if (skb1 && udp_queue_rcv_skb(sk, skb1) <= 0)
skb1 = NULL;
sock_put(sk);
}
if (unlikely(skb1))
kfree_skb(skb1);
}
/* For TCP sockets, sk_rx_dst is protected by socket lock /* For TCP sockets, sk_rx_dst is protected by socket lock
* For UDP, we use xchg() to guard against concurrent changes. * For UDP, we use xchg() to guard against concurrent changes.
*/ */
...@@ -1749,14 +1672,14 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, ...@@ -1749,14 +1672,14 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
struct udp_table *udptable, struct udp_table *udptable,
int proto) int proto)
{ {
struct sock *sk, *stack[256 / sizeof(struct sock *)]; struct sock *sk, *first = NULL;
struct hlist_nulls_node *node;
unsigned short hnum = ntohs(uh->dest); unsigned short hnum = ntohs(uh->dest);
struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum); struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
int dif = skb->dev->ifindex;
unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);
unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10); unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
bool inner_flushed = false; unsigned int offset = offsetof(typeof(*sk), sk_node);
int dif = skb->dev->ifindex;
struct hlist_node *node;
struct sk_buff *nskb;
if (use_hash2) { if (use_hash2) {
hash2_any = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum) & hash2_any = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
...@@ -1767,40 +1690,42 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, ...@@ -1767,40 +1690,42 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node); offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
} }
spin_lock(&hslot->lock); sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) { if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
if (__udp_is_mcast_sock(net, sk, uh->source, saddr, dif, hnum))
uh->dest, daddr, continue;
uh->source, saddr,
dif, hnum)) { if (!first) {
if (unlikely(count == ARRAY_SIZE(stack))) { first = sk;
flush_stack(stack, count, skb, ~0); continue;
inner_flushed = true;
count = 0;
} }
stack[count++] = sk; nskb = skb_clone(skb, GFP_ATOMIC);
sock_hold(sk);
if (unlikely(!nskb)) {
atomic_inc(&sk->sk_drops);
UDP_INC_STATS_BH(net, UDP_MIB_RCVBUFERRORS,
IS_UDPLITE(sk));
UDP_INC_STATS_BH(net, UDP_MIB_INERRORS,
IS_UDPLITE(sk));
continue;
} }
if (udp_queue_rcv_skb(sk, nskb) > 0)
consume_skb(nskb);
} }
spin_unlock(&hslot->lock);
/* Also lookup *:port if we are using hash2 and haven't done so yet. */ /* Also lookup *:port if we are using hash2 and haven't done so yet. */
if (use_hash2 && hash2 != hash2_any) { if (use_hash2 && hash2 != hash2_any) {
hash2 = hash2_any; hash2 = hash2_any;
goto start_lookup; goto start_lookup;
} }
/* if (first) {
* do the slow work with no lock held if (udp_queue_rcv_skb(first, skb) > 0)
*/ consume_skb(skb);
if (count) {
flush_stack(stack, count, skb, count - 1);
} else { } else {
if (!inner_flushed) kfree_skb(skb);
UDP_INC_STATS_BH(net, UDP_MIB_IGNOREDMULTI, UDP_INC_STATS_BH(net, UDP_MIB_IGNOREDMULTI,
proto == IPPROTO_UDPLITE); proto == IPPROTO_UDPLITE);
consume_skb(skb);
} }
return 0; return 0;
} }
...@@ -1897,7 +1822,6 @@ int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, ...@@ -1897,7 +1822,6 @@ int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
inet_compute_pseudo); inet_compute_pseudo);
ret = udp_queue_rcv_skb(sk, skb); ret = udp_queue_rcv_skb(sk, skb);
sock_put(sk);
/* a return value > 0 means to resubmit the input, but /* a return value > 0 means to resubmit the input, but
* it wants the return to be -protocol, or 0 * it wants the return to be -protocol, or 0
...@@ -1958,49 +1882,24 @@ static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net, ...@@ -1958,49 +1882,24 @@ static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
int dif) int dif)
{ {
struct sock *sk, *result; struct sock *sk, *result;
struct hlist_nulls_node *node;
unsigned short hnum = ntohs(loc_port); unsigned short hnum = ntohs(loc_port);
unsigned int count, slot = udp_hashfn(net, hnum, udp_table.mask); unsigned int slot = udp_hashfn(net, hnum, udp_table.mask);
struct udp_hslot *hslot = &udp_table.hash[slot]; struct udp_hslot *hslot = &udp_table.hash[slot];
/* Do not bother scanning a too big list */ /* Do not bother scanning a too big list */
if (hslot->count > 10) if (hslot->count > 10)
return NULL; return NULL;
rcu_read_lock();
begin:
count = 0;
result = NULL; result = NULL;
sk_nulls_for_each_rcu(sk, node, &hslot->head) { sk_for_each_rcu(sk, &hslot->head) {
if (__udp_is_mcast_sock(net, sk, if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
loc_port, loc_addr, rmt_port, rmt_addr, dif, hnum)) {
rmt_port, rmt_addr, if (result)
dif, hnum)) { return NULL;
result = sk; result = sk;
++count;
} }
} }
/*
* if the nulls value we got at the end of this lookup is
* not the expected one, we must restart lookup.
* We probably met an item that was moved to another chain.
*/
if (get_nulls_value(node) != slot)
goto begin;
if (result) {
if (count != 1 ||
unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
result = NULL;
else if (unlikely(!__udp_is_mcast_sock(net, result,
loc_port, loc_addr,
rmt_port, rmt_addr,
dif, hnum))) {
sock_put(result);
result = NULL;
}
}
rcu_read_unlock();
return result; return result;
} }
...@@ -2013,37 +1912,22 @@ static struct sock *__udp4_lib_demux_lookup(struct net *net, ...@@ -2013,37 +1912,22 @@ static struct sock *__udp4_lib_demux_lookup(struct net *net,
__be16 rmt_port, __be32 rmt_addr, __be16 rmt_port, __be32 rmt_addr,
int dif) int dif)
{ {
struct sock *sk, *result;
struct hlist_nulls_node *node;
unsigned short hnum = ntohs(loc_port); unsigned short hnum = ntohs(loc_port);
unsigned int hash2 = udp4_portaddr_hash(net, loc_addr, hnum); unsigned int hash2 = udp4_portaddr_hash(net, loc_addr, hnum);
unsigned int slot2 = hash2 & udp_table.mask; unsigned int slot2 = hash2 & udp_table.mask;
struct udp_hslot *hslot2 = &udp_table.hash2[slot2]; struct udp_hslot *hslot2 = &udp_table.hash2[slot2];
INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr); INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum); const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum);
struct sock *sk;
rcu_read_lock(); udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
result = NULL; if (INET_MATCH(sk, net, acookie, rmt_addr,
udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) { loc_addr, ports, dif))
if (INET_MATCH(sk, net, acookie, return sk;
rmt_addr, loc_addr, ports, dif))
result = sk;
/* Only check first socket in chain */ /* Only check first socket in chain */
break; break;
} }
return NULL;
if (result) {
if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
result = NULL;
else if (unlikely(!INET_MATCH(sk, net, acookie,
rmt_addr, loc_addr,
ports, dif))) {
sock_put(result);
result = NULL;
}
}
rcu_read_unlock();
return result;
} }
void udp_v4_early_demux(struct sk_buff *skb) void udp_v4_early_demux(struct sk_buff *skb)
...@@ -2051,7 +1935,7 @@ void udp_v4_early_demux(struct sk_buff *skb) ...@@ -2051,7 +1935,7 @@ void udp_v4_early_demux(struct sk_buff *skb)
struct net *net = dev_net(skb->dev); struct net *net = dev_net(skb->dev);
const struct iphdr *iph; const struct iphdr *iph;
const struct udphdr *uh; const struct udphdr *uh;
struct sock *sk; struct sock *sk = NULL;
struct dst_entry *dst; struct dst_entry *dst;
int dif = skb->dev->ifindex; int dif = skb->dev->ifindex;
int ours; int ours;
...@@ -2083,11 +1967,9 @@ void udp_v4_early_demux(struct sk_buff *skb) ...@@ -2083,11 +1967,9 @@ void udp_v4_early_demux(struct sk_buff *skb)
} else if (skb->pkt_type == PACKET_HOST) { } else if (skb->pkt_type == PACKET_HOST) {
sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr, sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
uh->source, iph->saddr, dif); uh->source, iph->saddr, dif);
} else {
return;
} }
if (!sk) if (!sk || !atomic_inc_not_zero_hint(&sk->sk_refcnt, 2))
return; return;
skb->sk = sk; skb->sk = sk;
...@@ -2387,14 +2269,13 @@ static struct sock *udp_get_first(struct seq_file *seq, int start) ...@@ -2387,14 +2269,13 @@ static struct sock *udp_get_first(struct seq_file *seq, int start)
for (state->bucket = start; state->bucket <= state->udp_table->mask; for (state->bucket = start; state->bucket <= state->udp_table->mask;
++state->bucket) { ++state->bucket) {
struct hlist_nulls_node *node;
struct udp_hslot *hslot = &state->udp_table->hash[state->bucket]; struct udp_hslot *hslot = &state->udp_table->hash[state->bucket];
if (hlist_nulls_empty(&hslot->head)) if (hlist_empty(&hslot->head))
continue; continue;
spin_lock_bh(&hslot->lock); spin_lock_bh(&hslot->lock);
sk_nulls_for_each(sk, node, &hslot->head) { sk_for_each(sk, &hslot->head) {
if (!net_eq(sock_net(sk), net)) if (!net_eq(sock_net(sk), net))
continue; continue;
if (sk->sk_family == state->family) if (sk->sk_family == state->family)
...@@ -2413,7 +2294,7 @@ static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk) ...@@ -2413,7 +2294,7 @@ static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
struct net *net = seq_file_net(seq); struct net *net = seq_file_net(seq);
do { do {
sk = sk_nulls_next(sk); sk = sk_next(sk);
} while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != state->family)); } while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != state->family));
if (!sk) { if (!sk) {
...@@ -2622,12 +2503,12 @@ void __init udp_table_init(struct udp_table *table, const char *name) ...@@ -2622,12 +2503,12 @@ void __init udp_table_init(struct udp_table *table, const char *name)
table->hash2 = table->hash + (table->mask + 1); table->hash2 = table->hash + (table->mask + 1);
for (i = 0; i <= table->mask; i++) { for (i = 0; i <= table->mask; i++) {
INIT_HLIST_NULLS_HEAD(&table->hash[i].head, i); INIT_HLIST_HEAD(&table->hash[i].head);
table->hash[i].count = 0; table->hash[i].count = 0;
spin_lock_init(&table->hash[i].lock); spin_lock_init(&table->hash[i].lock);
} }
for (i = 0; i <= table->mask; i++) { for (i = 0; i <= table->mask; i++) {
INIT_HLIST_NULLS_HEAD(&table->hash2[i].head, i); INIT_HLIST_HEAD(&table->hash2[i].head);
table->hash2[i].count = 0; table->hash2[i].count = 0;
spin_lock_init(&table->hash2[i].lock); spin_lock_init(&table->hash2[i].lock);
} }
......
net/ipv4/udp_diag.c
View file @ 15f41e2b
...@@ -36,10 +36,11 @@ static int udp_dump_one(struct udp_table *tbl, struct sk_buff *in_skb, ...@@ -36,10 +36,11 @@ static int udp_dump_one(struct udp_table *tbl, struct sk_buff *in_skb,
const struct inet_diag_req_v2 *req) const struct inet_diag_req_v2 *req)
{ {
int err = -EINVAL; int err = -EINVAL;
struct sock *sk; struct sock *sk = NULL;
struct sk_buff *rep; struct sk_buff *rep;
struct net *net = sock_net(in_skb->sk); struct net *net = sock_net(in_skb->sk);
rcu_read_lock();
if (req->sdiag_family == AF_INET) if (req->sdiag_family == AF_INET)
sk = __udp4_lib_lookup(net, sk = __udp4_lib_lookup(net,
req->id.idiag_src[0], req->id.idiag_sport, req->id.idiag_src[0], req->id.idiag_sport,
...@@ -54,9 +55,9 @@ static int udp_dump_one(struct udp_table *tbl, struct sk_buff *in_skb, ...@@ -54,9 +55,9 @@ static int udp_dump_one(struct udp_table *tbl, struct sk_buff *in_skb,
req->id.idiag_dport, req->id.idiag_dport,
req->id.idiag_if, tbl, NULL); req->id.idiag_if, tbl, NULL);
#endif #endif
else if (sk && !atomic_inc_not_zero(&sk->sk_refcnt))
goto out_nosk; sk = NULL;
rcu_read_unlock();
err = -ENOENT; err = -ENOENT;
if (!sk) if (!sk)
goto out_nosk; goto out_nosk;
...@@ -96,24 +97,23 @@ static void udp_dump(struct udp_table *table, struct sk_buff *skb, ...@@ -96,24 +97,23 @@ static void udp_dump(struct udp_table *table, struct sk_buff *skb,
struct netlink_callback *cb, struct netlink_callback *cb,
const struct inet_diag_req_v2 *r, struct nlattr *bc) const struct inet_diag_req_v2 *r, struct nlattr *bc)
{ {
int num, s_num, slot, s_slot;
struct net *net = sock_net(skb->sk); struct net *net = sock_net(skb->sk);
int num, s_num, slot, s_slot;
s_slot = cb->args[0]; s_slot = cb->args[0];
num = s_num = cb->args[1]; num = s_num = cb->args[1];
for (slot = s_slot; slot <= table->mask; s_num = 0, slot++) { for (slot = s_slot; slot <= table->mask; s_num = 0, slot++) {
struct sock *sk;
struct hlist_nulls_node *node;
struct udp_hslot *hslot = &table->hash[slot]; struct udp_hslot *hslot = &table->hash[slot];
struct sock *sk;
num = 0; num = 0;
if (hlist_nulls_empty(&hslot->head)) if (hlist_empty(&hslot->head))
continue; continue;
spin_lock_bh(&hslot->lock); spin_lock_bh(&hslot->lock);
sk_nulls_for_each(sk, node, &hslot->head) { sk_for_each(sk, &hslot->head) {
struct inet_sock *inet = inet_sk(sk); struct inet_sock *inet = inet_sk(sk);
if (!net_eq(sock_net(sk), net)) if (!net_eq(sock_net(sk), net))
......
net/ipv6/inet6_hashtables.c
View file @ 15f41e2b
...@@ -69,7 +69,6 @@ struct sock *__inet6_lookup_established(struct net *net, ...@@ -69,7 +69,6 @@ struct sock *__inet6_lookup_established(struct net *net,
struct inet_ehash_bucket *head = &hashinfo->ehash[slot]; struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
rcu_read_lock();
begin: begin:
sk_nulls_for_each_rcu(sk, node, &head->chain) { sk_nulls_for_each_rcu(sk, node, &head->chain) {
if (sk->sk_hash != hash) if (sk->sk_hash != hash)
...@@ -90,7 +89,6 @@ struct sock *__inet6_lookup_established(struct net *net, ...@@ -90,7 +89,6 @@ struct sock *__inet6_lookup_established(struct net *net,
out: out:
sk = NULL; sk = NULL;
found: found:
rcu_read_unlock();
return sk; return sk;
} }
EXPORT_SYMBOL(__inet6_lookup_established); EXPORT_SYMBOL(__inet6_lookup_established);
...@@ -122,6 +120,7 @@ static inline int compute_score(struct sock *sk, struct net *net, ...@@ -122,6 +120,7 @@ static inline int compute_score(struct sock *sk, struct net *net,
return score; return score;
} }
/* called with rcu_read_lock() */
struct sock *inet6_lookup_listener(struct net *net, struct sock *inet6_lookup_listener(struct net *net,
struct inet_hashinfo *hashinfo, struct inet_hashinfo *hashinfo,
struct sk_buff *skb, int doff, struct sk_buff *skb, int doff,
...@@ -129,39 +128,27 @@ struct sock *inet6_lookup_listener(struct net *net, ...@@ -129,39 +128,27 @@ struct sock *inet6_lookup_listener(struct net *net,
const __be16 sport, const struct in6_addr *daddr, const __be16 sport, const struct in6_addr *daddr,
const unsigned short hnum, const int dif) const unsigned short hnum, const int dif)
{ {
struct sock *sk;
const struct hlist_nulls_node *node;
struct sock *result;
int score, hiscore, matches = 0, reuseport = 0;
bool select_ok = true;
u32 phash = 0;
unsigned int hash = inet_lhashfn(net, hnum); unsigned int hash = inet_lhashfn(net, hnum);
struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
int score, hiscore = 0, matches = 0, reuseport = 0;
struct sock *sk, *result = NULL;
u32 phash = 0;
rcu_read_lock(); sk_for_each(sk, &ilb->head) {
begin:
result = NULL;
hiscore = 0;
sk_nulls_for_each(sk, node, &ilb->head) {
score = compute_score(sk, net, hnum, daddr, dif); score = compute_score(sk, net, hnum, daddr, dif);
if (score > hiscore) { if (score > hiscore) {
hiscore = score; hiscore = score;
result = sk;
reuseport = sk->sk_reuseport;
if (reuseport) { if (reuseport) {
phash = inet6_ehashfn(net, daddr, hnum, phash = inet6_ehashfn(net, daddr, hnum,
saddr, sport); saddr, sport);
if (select_ok) { result = reuseport_select_sock(sk, phash,
struct sock *sk2;
sk2 = reuseport_select_sock(sk, phash,
skb, doff); skb, doff);
if (sk2) { if (result)
result = sk2; return result;
goto found;
}
}
matches = 1; matches = 1;
} }
result = sk;
reuseport = sk->sk_reuseport;
} else if (score == hiscore && reuseport) { } else if (score == hiscore && reuseport) {
matches++; matches++;
if (reciprocal_scale(phash, matches) == 0) if (reciprocal_scale(phash, matches) == 0)
...@@ -169,25 +156,6 @@ struct sock *inet6_lookup_listener(struct net *net, ...@@ -169,25 +156,6 @@ struct sock *inet6_lookup_listener(struct net *net,
phash = next_pseudo_random32(phash); phash = next_pseudo_random32(phash);
} }
} }
/*
* if the nulls value we got at the end of this lookup is
* not the expected one, we must restart lookup.
* We probably met an item that was moved to another chain.
*/
if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
goto begin;
if (result) {
found:
if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
result = NULL;
else if (unlikely(compute_score(result, net, hnum, daddr,
dif) < hiscore)) {
sock_put(result);
select_ok = false;
goto begin;
}
}
rcu_read_unlock();
return result; return result;
} }
EXPORT_SYMBOL_GPL(inet6_lookup_listener); EXPORT_SYMBOL_GPL(inet6_lookup_listener);
...@@ -199,12 +167,12 @@ struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo, ...@@ -199,12 +167,12 @@ struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
const int dif) const int dif)
{ {
struct sock *sk; struct sock *sk;
bool refcounted;
local_bh_disable();
sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr, sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr,
ntohs(dport), dif); ntohs(dport), dif, &refcounted);
local_bh_enable(); if (sk && !refcounted && !atomic_inc_not_zero(&sk->sk_refcnt))
sk = NULL;
return sk; return sk;
} }
EXPORT_SYMBOL_GPL(inet6_lookup); EXPORT_SYMBOL_GPL(inet6_lookup);
......
net/ipv6/tcp_ipv6.c
View file @ 15f41e2b
...@@ -858,6 +858,7 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb) ...@@ -858,6 +858,7 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
return; return;
#ifdef CONFIG_TCP_MD5SIG #ifdef CONFIG_TCP_MD5SIG
rcu_read_lock();
hash_location = tcp_parse_md5sig_option(th); hash_location = tcp_parse_md5sig_option(th);
if (sk && sk_fullsock(sk)) { if (sk && sk_fullsock(sk)) {
key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr); key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr);
...@@ -875,16 +876,15 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb) ...@@ -875,16 +876,15 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
th->source, &ipv6h->daddr, th->source, &ipv6h->daddr,
ntohs(th->source), tcp_v6_iif(skb)); ntohs(th->source), tcp_v6_iif(skb));
if (!sk1) if (!sk1)
return; goto out;
rcu_read_lock();
key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr); key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr);
if (!key) if (!key)
goto release_sk1; goto out;
genhash = tcp_v6_md5_hash_skb(newhash, key, NULL, skb); genhash = tcp_v6_md5_hash_skb(newhash, key, NULL, skb);
if (genhash || memcmp(hash_location, newhash, 16) != 0) if (genhash || memcmp(hash_location, newhash, 16) != 0)
goto release_sk1; goto out;
} }
#endif #endif
...@@ -898,11 +898,8 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb) ...@@ -898,11 +898,8 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, key, 1, 0, 0); tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, key, 1, 0, 0);
#ifdef CONFIG_TCP_MD5SIG #ifdef CONFIG_TCP_MD5SIG
release_sk1: out:
if (sk1) {
rcu_read_unlock(); rcu_read_unlock();
sock_put(sk1);
}
#endif #endif
} }
...@@ -967,7 +964,7 @@ static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb) ...@@ -967,7 +964,7 @@ static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
&tcp_request_sock_ipv6_ops, sk, skb); &tcp_request_sock_ipv6_ops, sk, skb);
drop: drop:
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS); tcp_listendrop(sk);
return 0; /* don't send reset */ return 0; /* don't send reset */
} }
...@@ -1172,7 +1169,7 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * ...@@ -1172,7 +1169,7 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
out_nonewsk: out_nonewsk:
dst_release(dst); dst_release(dst);
out: out:
NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS); tcp_listendrop(sk);
return NULL; return NULL;
} }
...@@ -1351,6 +1348,7 @@ static int tcp_v6_rcv(struct sk_buff *skb) ...@@ -1351,6 +1348,7 @@ static int tcp_v6_rcv(struct sk_buff *skb)
{ {
const struct tcphdr *th; const struct tcphdr *th;
const struct ipv6hdr *hdr; const struct ipv6hdr *hdr;
bool refcounted;
struct sock *sk; struct sock *sk;
int ret; int ret;
struct net *net = dev_net(skb->dev); struct net *net = dev_net(skb->dev);
...@@ -1381,7 +1379,8 @@ static int tcp_v6_rcv(struct sk_buff *skb) ...@@ -1381,7 +1379,8 @@ static int tcp_v6_rcv(struct sk_buff *skb)
lookup: lookup:
sk = __inet6_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), sk = __inet6_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th),
th->source, th->dest, inet6_iif(skb)); th->source, th->dest, inet6_iif(skb),
&refcounted);
if (!sk) if (!sk)
goto no_tcp_socket; goto no_tcp_socket;
...@@ -1404,6 +1403,7 @@ static int tcp_v6_rcv(struct sk_buff *skb) ...@@ -1404,6 +1403,7 @@ static int tcp_v6_rcv(struct sk_buff *skb)
goto lookup; goto lookup;
} }
sock_hold(sk); sock_hold(sk);
refcounted = true;
nsk = tcp_check_req(sk, skb, req, false); nsk = tcp_check_req(sk, skb, req, false);
if (!nsk) { if (!nsk) {
reqsk_put(req); reqsk_put(req);
...@@ -1460,6 +1460,7 @@ static int tcp_v6_rcv(struct sk_buff *skb) ...@@ -1460,6 +1460,7 @@ static int tcp_v6_rcv(struct sk_buff *skb)
bh_unlock_sock(sk); bh_unlock_sock(sk);
put_and_return: put_and_return:
if (refcounted)
sock_put(sk); sock_put(sk);
return ret ? -1 : 0; return ret ? -1 : 0;
...@@ -1483,6 +1484,8 @@ static int tcp_v6_rcv(struct sk_buff *skb) ...@@ -1483,6 +1484,8 @@ static int tcp_v6_rcv(struct sk_buff *skb)
return 0; return 0;
discard_and_relse: discard_and_relse:
sk_drops_add(sk, skb);
if (refcounted)
sock_put(sk); sock_put(sk);
goto discard_it; goto discard_it;
...@@ -1514,6 +1517,7 @@ static int tcp_v6_rcv(struct sk_buff *skb) ...@@ -1514,6 +1517,7 @@ static int tcp_v6_rcv(struct sk_buff *skb)
inet_twsk_deschedule_put(tw); inet_twsk_deschedule_put(tw);
sk = sk2; sk = sk2;
tcp_v6_restore_cb(skb); tcp_v6_restore_cb(skb);
refcounted = false;
goto process; goto process;
} }
/* Fall through to ACK */ /* Fall through to ACK */
......
net/ipv6/udp.c
View file @ 15f41e2b
...@@ -213,37 +213,28 @@ static struct sock *udp6_lib_lookup2(struct net *net, ...@@ -213,37 +213,28 @@ static struct sock *udp6_lib_lookup2(struct net *net,
struct sk_buff *skb) struct sk_buff *skb)
{ {
struct sock *sk, *result; struct sock *sk, *result;
struct hlist_nulls_node *node;
int score, badness, matches = 0, reuseport = 0; int score, badness, matches = 0, reuseport = 0;
bool select_ok = true;
u32 hash = 0; u32 hash = 0;
begin:
result = NULL; result = NULL;
badness = -1; badness = -1;
udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) { udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
score = compute_score2(sk, net, saddr, sport, score = compute_score2(sk, net, saddr, sport,
daddr, hnum, dif); daddr, hnum, dif);
if (score > badness) { if (score > badness) {
result = sk;
badness = score;
reuseport = sk->sk_reuseport; reuseport = sk->sk_reuseport;
if (reuseport) { if (reuseport) {
hash = udp6_ehashfn(net, daddr, hnum, hash = udp6_ehashfn(net, daddr, hnum,
saddr, sport); saddr, sport);
if (select_ok) {
struct sock *sk2;
sk2 = reuseport_select_sock(sk, hash, skb, result = reuseport_select_sock(sk, hash, skb,
sizeof(struct udphdr)); sizeof(struct udphdr));
if (sk2) { if (result)
result = sk2; return result;
select_ok = false;
goto found;
}
}
matches = 1; matches = 1;
} }
result = sk;
badness = score;
} else if (score == badness && reuseport) { } else if (score == badness && reuseport) {
matches++; matches++;
if (reciprocal_scale(hash, matches) == 0) if (reciprocal_scale(hash, matches) == 0)
...@@ -251,27 +242,10 @@ static struct sock *udp6_lib_lookup2(struct net *net, ...@@ -251,27 +242,10 @@ static struct sock *udp6_lib_lookup2(struct net *net,
hash = next_pseudo_random32(hash); hash = next_pseudo_random32(hash);
} }
} }
/*
* if the nulls value we got at the end of this lookup is
* not the expected one, we must restart lookup.
* We probably met an item that was moved to another chain.
*/
if (get_nulls_value(node) != slot2)
goto begin;
if (result) {
found:
if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
result = NULL;
else if (unlikely(compute_score2(result, net, saddr, sport,
daddr, hnum, dif) < badness)) {
sock_put(result);
goto begin;
}
}
return result; return result;
} }
/* rcu_read_lock() must be held */
struct sock *__udp6_lib_lookup(struct net *net, struct sock *__udp6_lib_lookup(struct net *net,
const struct in6_addr *saddr, __be16 sport, const struct in6_addr *saddr, __be16 sport,
const struct in6_addr *daddr, __be16 dport, const struct in6_addr *daddr, __be16 dport,
...@@ -279,15 +253,12 @@ struct sock *__udp6_lib_lookup(struct net *net, ...@@ -279,15 +253,12 @@ struct sock *__udp6_lib_lookup(struct net *net,
struct sk_buff *skb) struct sk_buff *skb)
{ {
struct sock *sk, *result; struct sock *sk, *result;
struct hlist_nulls_node *node;
unsigned short hnum = ntohs(dport); unsigned short hnum = ntohs(dport);
unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask); unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
struct udp_hslot *hslot2, *hslot = &udptable->hash[slot]; struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
int score, badness, matches = 0, reuseport = 0; int score, badness, matches = 0, reuseport = 0;
bool select_ok = true;
u32 hash = 0; u32 hash = 0;
rcu_read_lock();
if (hslot->count > 10) { if (hslot->count > 10) {
hash2 = udp6_portaddr_hash(net, daddr, hnum); hash2 = udp6_portaddr_hash(net, daddr, hnum);
slot2 = hash2 & udptable->mask; slot2 = hash2 & udptable->mask;
...@@ -309,34 +280,26 @@ struct sock *__udp6_lib_lookup(struct net *net, ...@@ -309,34 +280,26 @@ struct sock *__udp6_lib_lookup(struct net *net,
&in6addr_any, hnum, dif, &in6addr_any, hnum, dif,
hslot2, slot2, skb); hslot2, slot2, skb);
} }
rcu_read_unlock();
return result; return result;
} }
begin: begin:
result = NULL; result = NULL;
badness = -1; badness = -1;
sk_nulls_for_each_rcu(sk, node, &hslot->head) { sk_for_each_rcu(sk, &hslot->head) {
score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif); score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif);
if (score > badness) { if (score > badness) {
result = sk;
badness = score;
reuseport = sk->sk_reuseport; reuseport = sk->sk_reuseport;
if (reuseport) { if (reuseport) {
hash = udp6_ehashfn(net, daddr, hnum, hash = udp6_ehashfn(net, daddr, hnum,
saddr, sport); saddr, sport);
if (select_ok) { result = reuseport_select_sock(sk, hash, skb,
struct sock *sk2;
sk2 = reuseport_select_sock(sk, hash, skb,
sizeof(struct udphdr)); sizeof(struct udphdr));
if (sk2) { if (result)
result = sk2; return result;
select_ok = false;
goto found;
}
}
matches = 1; matches = 1;
} }
result = sk;
badness = score;
} else if (score == badness && reuseport) { } else if (score == badness && reuseport) {
matches++; matches++;
if (reciprocal_scale(hash, matches) == 0) if (reciprocal_scale(hash, matches) == 0)
...@@ -344,25 +307,6 @@ struct sock *__udp6_lib_lookup(struct net *net, ...@@ -344,25 +307,6 @@ struct sock *__udp6_lib_lookup(struct net *net,
hash = next_pseudo_random32(hash); hash = next_pseudo_random32(hash);
} }
} }
/*
* if the nulls value we got at the end of this lookup is
* not the expected one, we must restart lookup.
* We probably met an item that was moved to another chain.
*/
if (get_nulls_value(node) != slot)
goto begin;
if (result) {
found:
if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
result = NULL;
else if (unlikely(compute_score(result, net, hnum, saddr, sport,
daddr, dport, dif) < badness)) {
sock_put(result);
goto begin;
}
}
rcu_read_unlock();
return result; return result;
} }
EXPORT_SYMBOL_GPL(__udp6_lib_lookup); EXPORT_SYMBOL_GPL(__udp6_lib_lookup);
...@@ -382,12 +326,24 @@ static struct sock *__udp6_lib_lookup_skb(struct sk_buff *skb, ...@@ -382,12 +326,24 @@ static struct sock *__udp6_lib_lookup_skb(struct sk_buff *skb,
udptable, skb); udptable, skb);
} }
/* Must be called under rcu_read_lock().
* Does increment socket refcount.
*/
#if IS_ENABLED(CONFIG_NETFILTER_XT_MATCH_SOCKET) || \
IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TPROXY)
struct sock *udp6_lib_lookup(struct net *net, const struct in6_addr *saddr, __be16 sport, struct sock *udp6_lib_lookup(struct net *net, const struct in6_addr *saddr, __be16 sport,
const struct in6_addr *daddr, __be16 dport, int dif) const struct in6_addr *daddr, __be16 dport, int dif)
{ {
return __udp6_lib_lookup(net, saddr, sport, daddr, dport, dif, &udp_table, NULL); struct sock *sk;
sk = __udp6_lib_lookup(net, saddr, sport, daddr, dport,
dif, &udp_table, NULL);
if (sk && !atomic_inc_not_zero(&sk->sk_refcnt))
sk = NULL;
return sk;
} }
EXPORT_SYMBOL_GPL(udp6_lib_lookup); EXPORT_SYMBOL_GPL(udp6_lib_lookup);
#endif
/* /*
* This should be easy, if there is something there we * This should be easy, if there is something there we
...@@ -585,7 +541,7 @@ void __udp6_lib_err(struct sk_buff *skb, struct inet6_skb_parm *opt, ...@@ -585,7 +541,7 @@ void __udp6_lib_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
sk->sk_err = err; sk->sk_err = err;
sk->sk_error_report(sk); sk->sk_error_report(sk);
out: out:
sock_put(sk); return;
} }
static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
...@@ -747,33 +703,6 @@ static bool __udp_v6_is_mcast_sock(struct net *net, struct sock *sk, ...@@ -747,33 +703,6 @@ static bool __udp_v6_is_mcast_sock(struct net *net, struct sock *sk,
return true; return true;
} }
static void flush_stack(struct sock **stack, unsigned int count,
struct sk_buff *skb, unsigned int final)
{
struct sk_buff *skb1 = NULL;
struct sock *sk;
unsigned int i;
for (i = 0; i < count; i++) {
sk = stack[i];
if (likely(!skb1))
skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
if (!skb1) {
atomic_inc(&sk->sk_drops);
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
IS_UDPLITE(sk));
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
IS_UDPLITE(sk));
}
if (skb1 && udpv6_queue_rcv_skb(sk, skb1) <= 0)
skb1 = NULL;
sock_put(sk);
}
if (unlikely(skb1))
kfree_skb(skb1);
}
static void udp6_csum_zero_error(struct sk_buff *skb) static void udp6_csum_zero_error(struct sk_buff *skb)
{ {
/* RFC 2460 section 8.1 says that we SHOULD log /* RFC 2460 section 8.1 says that we SHOULD log
...@@ -792,15 +721,15 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb, ...@@ -792,15 +721,15 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
const struct in6_addr *saddr, const struct in6_addr *daddr, const struct in6_addr *saddr, const struct in6_addr *daddr,
struct udp_table *udptable, int proto) struct udp_table *udptable, int proto)
{ {
struct sock *sk, *stack[256 / sizeof(struct sock *)]; struct sock *sk, *first = NULL;
const struct udphdr *uh = udp_hdr(skb); const struct udphdr *uh = udp_hdr(skb);
struct hlist_nulls_node *node;
unsigned short hnum = ntohs(uh->dest); unsigned short hnum = ntohs(uh->dest);
struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum); struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
int dif = inet6_iif(skb); unsigned int offset = offsetof(typeof(*sk), sk_node);
unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);
unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10); unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
bool inner_flushed = false; int dif = inet6_iif(skb);
struct hlist_node *node;
struct sk_buff *nskb;
if (use_hash2) { if (use_hash2) {
hash2_any = udp6_portaddr_hash(net, &in6addr_any, hnum) & hash2_any = udp6_portaddr_hash(net, &in6addr_any, hnum) &
...@@ -811,27 +740,32 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb, ...@@ -811,27 +740,32 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node); offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
} }
spin_lock(&hslot->lock); sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) { if (!__udp_v6_is_mcast_sock(net, sk, uh->dest, daddr,
if (__udp_v6_is_mcast_sock(net, sk, uh->source, saddr, dif, hnum))
uh->dest, daddr, continue;
uh->source, saddr,
dif, hnum) &&
/* If zero checksum and no_check is not on for /* If zero checksum and no_check is not on for
* the socket then skip it. * the socket then skip it.
*/ */
(uh->check || udp_sk(sk)->no_check6_rx)) { if (!uh->check && !udp_sk(sk)->no_check6_rx)
if (unlikely(count == ARRAY_SIZE(stack))) { continue;
flush_stack(stack, count, skb, ~0); if (!first) {
inner_flushed = true; first = sk;
count = 0; continue;
} }
stack[count++] = sk; nskb = skb_clone(skb, GFP_ATOMIC);
sock_hold(sk); if (unlikely(!nskb)) {
} atomic_inc(&sk->sk_drops);
UDP6_INC_STATS_BH(net, UDP_MIB_RCVBUFERRORS,
IS_UDPLITE(sk));
UDP6_INC_STATS_BH(net, UDP_MIB_INERRORS,
IS_UDPLITE(sk));
continue;
} }
spin_unlock(&hslot->lock); if (udpv6_queue_rcv_skb(sk, nskb) > 0)
consume_skb(nskb);
}
/* Also lookup *:port if we are using hash2 and haven't done so yet. */ /* Also lookup *:port if we are using hash2 and haven't done so yet. */
if (use_hash2 && hash2 != hash2_any) { if (use_hash2 && hash2 != hash2_any) {
...@@ -839,13 +773,13 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb, ...@@ -839,13 +773,13 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
goto start_lookup; goto start_lookup;
} }
if (count) { if (first) {
flush_stack(stack, count, skb, count - 1); if (udpv6_queue_rcv_skb(first, skb) > 0)
consume_skb(skb);
} else { } else {
if (!inner_flushed) kfree_skb(skb);
UDP6_INC_STATS_BH(net, UDP_MIB_IGNOREDMULTI, UDP6_INC_STATS_BH(net, UDP_MIB_IGNOREDMULTI,
proto == IPPROTO_UDPLITE); proto == IPPROTO_UDPLITE);
consume_skb(skb);
} }
return 0; return 0;
} }
...@@ -853,10 +787,10 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb, ...@@ -853,10 +787,10 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
int proto) int proto)
{ {
const struct in6_addr *saddr, *daddr;
struct net *net = dev_net(skb->dev); struct net *net = dev_net(skb->dev);
struct sock *sk;
struct udphdr *uh; struct udphdr *uh;
const struct in6_addr *saddr, *daddr; struct sock *sk;
u32 ulen = 0; u32 ulen = 0;
if (!pskb_may_pull(skb, sizeof(struct udphdr))) if (!pskb_may_pull(skb, sizeof(struct udphdr)))
...@@ -910,7 +844,6 @@ int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, ...@@ -910,7 +844,6 @@ int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
int ret; int ret;
if (!uh->check && !udp_sk(sk)->no_check6_rx) { if (!uh->check && !udp_sk(sk)->no_check6_rx) {
sock_put(sk);
udp6_csum_zero_error(skb); udp6_csum_zero_error(skb);
goto csum_error; goto csum_error;
} }
...@@ -920,7 +853,6 @@ int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, ...@@ -920,7 +853,6 @@ int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
ip6_compute_pseudo); ip6_compute_pseudo);
ret = udpv6_queue_rcv_skb(sk, skb); ret = udpv6_queue_rcv_skb(sk, skb);
sock_put(sk);
/* a return value > 0 means to resubmit the input */ /* a return value > 0 means to resubmit the input */
if (ret > 0) if (ret > 0)
......
net/netfilter/xt_socket.c
View file @ 15f41e2b
...@@ -120,7 +120,7 @@ xt_socket_get_sock_v4(struct net *net, struct sk_buff *skb, const int doff, ...@@ -120,7 +120,7 @@ xt_socket_get_sock_v4(struct net *net, struct sk_buff *skb, const int doff,
{ {
switch (protocol) { switch (protocol) {
case IPPROTO_TCP: case IPPROTO_TCP:
return __inet_lookup(net, &tcp_hashinfo, skb, doff, return inet_lookup(net, &tcp_hashinfo, skb, doff,
saddr, sport, daddr, dport, saddr, sport, daddr, dport,
in->ifindex); in->ifindex);
case IPPROTO_UDP: case IPPROTO_UDP:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7