Commit abaeb096 authored by John Fastabend's avatar John Fastabend Committed by Alexei Starovoitov

bpf: sockmap, fix error handling in redirect failures

When a redirect failure happens we release the buffers in-flight
without calling a sk_mem_uncharge(), the uncharge is called before
dropping the sock lock for the redirecte, however we missed updating
the ring start index. When no apply actions are in progress this
is OK because we uncharge the entire buffer before the redirect.
But, when we have apply logic running its possible that only a
portion of the buffer is being redirected. In this case we only
do memory accounting for the buffer slice being redirected and
expect to be able to loop over the BPF program again and/or if
a sock is closed uncharge the memory at sock destruct time.

With an invalid start index however the program logic looks at
the start pointer index, checks the length, and when seeing the
length is zero (from the initial release and failure to update
the pointer) aborts without uncharging/releasing the remaining
memory.

The fix for this is simply to update the start index. To avoid
fixing this error in two locations we do a small refactor and
remove one case where it is open-coded. Then fix it in the
single function.
Signed-off-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent fec51d40
...@@ -393,7 +393,8 @@ static void return_mem_sg(struct sock *sk, int bytes, struct sk_msg_buff *md) ...@@ -393,7 +393,8 @@ static void return_mem_sg(struct sock *sk, int bytes, struct sk_msg_buff *md)
} while (i != md->sg_end); } while (i != md->sg_end);
} }
static void free_bytes_sg(struct sock *sk, int bytes, struct sk_msg_buff *md) static void free_bytes_sg(struct sock *sk, int bytes,
struct sk_msg_buff *md, bool charge)
{ {
struct scatterlist *sg = md->sg_data; struct scatterlist *sg = md->sg_data;
int i = md->sg_start, free; int i = md->sg_start, free;
...@@ -403,10 +404,12 @@ static void free_bytes_sg(struct sock *sk, int bytes, struct sk_msg_buff *md) ...@@ -403,10 +404,12 @@ static void free_bytes_sg(struct sock *sk, int bytes, struct sk_msg_buff *md)
if (bytes < free) { if (bytes < free) {
sg[i].length -= bytes; sg[i].length -= bytes;
sg[i].offset += bytes; sg[i].offset += bytes;
if (charge)
sk_mem_uncharge(sk, bytes); sk_mem_uncharge(sk, bytes);
break; break;
} }
if (charge)
sk_mem_uncharge(sk, sg[i].length); sk_mem_uncharge(sk, sg[i].length);
put_page(sg_page(&sg[i])); put_page(sg_page(&sg[i]));
bytes -= sg[i].length; bytes -= sg[i].length;
...@@ -418,6 +421,7 @@ static void free_bytes_sg(struct sock *sk, int bytes, struct sk_msg_buff *md) ...@@ -418,6 +421,7 @@ static void free_bytes_sg(struct sock *sk, int bytes, struct sk_msg_buff *md)
if (i == MAX_SKB_FRAGS) if (i == MAX_SKB_FRAGS)
i = 0; i = 0;
} }
md->sg_start = i;
} }
static int free_sg(struct sock *sk, int start, struct sk_msg_buff *md) static int free_sg(struct sock *sk, int start, struct sk_msg_buff *md)
...@@ -576,10 +580,10 @@ static int bpf_tcp_sendmsg_do_redirect(struct sock *sk, int send, ...@@ -576,10 +580,10 @@ static int bpf_tcp_sendmsg_do_redirect(struct sock *sk, int send,
struct sk_msg_buff *md, struct sk_msg_buff *md,
int flags) int flags)
{ {
bool ingress = !!(md->flags & BPF_F_INGRESS);
struct smap_psock *psock; struct smap_psock *psock;
struct scatterlist *sg; struct scatterlist *sg;
int i, err, free = 0; int err = 0;
bool ingress = !!(md->flags & BPF_F_INGRESS);
sg = md->sg_data; sg = md->sg_data;
...@@ -607,16 +611,8 @@ static int bpf_tcp_sendmsg_do_redirect(struct sock *sk, int send, ...@@ -607,16 +611,8 @@ static int bpf_tcp_sendmsg_do_redirect(struct sock *sk, int send,
out_rcu: out_rcu:
rcu_read_unlock(); rcu_read_unlock();
out: out:
i = md->sg_start; free_bytes_sg(NULL, send, md, false);
while (sg[i].length) { return err;
free += sg[i].length;
put_page(sg_page(&sg[i]));
sg[i].length = 0;
i++;
if (i == MAX_SKB_FRAGS)
i = 0;
}
return free;
} }
static inline void bpf_md_init(struct smap_psock *psock) static inline void bpf_md_init(struct smap_psock *psock)
...@@ -720,7 +716,7 @@ static int bpf_exec_tx_verdict(struct smap_psock *psock, ...@@ -720,7 +716,7 @@ static int bpf_exec_tx_verdict(struct smap_psock *psock,
break; break;
case __SK_DROP: case __SK_DROP:
default: default:
free_bytes_sg(sk, send, m); free_bytes_sg(sk, send, m, true);
apply_bytes_dec(psock, send); apply_bytes_dec(psock, send);
*copied -= send; *copied -= send;
psock->sg_size -= send; psock->sg_size -= send;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment