Commit d829e9c4 authored by Daniel Borkmann's avatar Daniel Borkmann Committed by Alexei Starovoitov

tls: convert to generic sk_msg interface

Convert kTLS over to make use of sk_msg interface for plaintext and
encrypted scattergather data, so it reuses all the sk_msg helpers
and data structure which later on in a second step enables to glue
this to BPF.

This also allows to remove quite a bit of open coded helpers which
are covered by the sk_msg API. Recent changes in kTLs 80ece6a0
("tls: Remove redundant vars from tls record structure") and
4e6d4720 ("tls: Add support for inplace records encryption")
changed the data path handling a bit; while we've kept the latter
optimization intact, we had to undo the former change to better
fit the sk_msg model, hence the sg_aead_in and sg_aead_out have
been brought back and are linked into the sk_msg sgs. Now the kTLS
record contains a msg_plaintext and msg_encrypted sk_msg each.

In the original code, the zerocopy_from_iter() has been used out
of TX but also RX path. For the strparser skb-based RX path,
we've left the zerocopy_from_iter() in decrypt_internal() mostly
untouched, meaning it has been moved into tls_setup_from_iter()
with charging logic removed (as not used from RX). Given RX path
is not based on sk_msg objects, we haven't pursued setting up a
dummy sk_msg to call into sk_msg_zerocopy_from_iter(), but it
could be an option to prusue in a later step.

Joint work with John.
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Signed-off-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 604326b4
...@@ -102,6 +102,8 @@ struct sk_psock { ...@@ -102,6 +102,8 @@ struct sk_psock {
int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
int elem_first_coalesce); int elem_first_coalesce);
int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
u32 off, u32 len);
void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len); void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
int sk_msg_free(struct sock *sk, struct sk_msg *msg); int sk_msg_free(struct sock *sk, struct sk_msg *msg);
int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg); int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
......
...@@ -2214,10 +2214,6 @@ static inline struct page_frag *sk_page_frag(struct sock *sk) ...@@ -2214,10 +2214,6 @@ static inline struct page_frag *sk_page_frag(struct sock *sk)
bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag); bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
int sk_alloc_sg(struct sock *sk, int len, struct scatterlist *sg,
int sg_start, int *sg_curr, unsigned int *sg_size,
int first_coalesce);
/* /*
* Default write policy as shown to user space via poll/select/SIGIO * Default write policy as shown to user space via poll/select/SIGIO
*/ */
......
...@@ -39,6 +39,8 @@ ...@@ -39,6 +39,8 @@
#include <linux/crypto.h> #include <linux/crypto.h>
#include <linux/socket.h> #include <linux/socket.h>
#include <linux/tcp.h> #include <linux/tcp.h>
#include <linux/skmsg.h>
#include <net/tcp.h> #include <net/tcp.h>
#include <net/strparser.h> #include <net/strparser.h>
#include <crypto/aead.h> #include <crypto/aead.h>
...@@ -103,15 +105,13 @@ struct tls_rec { ...@@ -103,15 +105,13 @@ struct tls_rec {
int tx_flags; int tx_flags;
int inplace_crypto; int inplace_crypto;
/* AAD | sg_plaintext_data | sg_tag */ struct sk_msg msg_plaintext;
struct scatterlist sg_plaintext_data[MAX_SKB_FRAGS + 1]; struct sk_msg msg_encrypted;
/* AAD | sg_encrypted_data (data contain overhead for hdr&iv&tag) */
struct scatterlist sg_encrypted_data[MAX_SKB_FRAGS + 1];
unsigned int sg_plaintext_size; /* AAD | msg_plaintext.sg.data | sg_tag */
unsigned int sg_encrypted_size; struct scatterlist sg_aead_in[2];
int sg_plaintext_num_elem; /* AAD | msg_encrypted.sg.data (data contains overhead for hdr & iv & tag) */
int sg_encrypted_num_elem; struct scatterlist sg_aead_out[2];
char aad_space[TLS_AAD_SPACE_SIZE]; char aad_space[TLS_AAD_SPACE_SIZE];
struct aead_request aead_req; struct aead_request aead_req;
...@@ -223,8 +223,8 @@ struct tls_context { ...@@ -223,8 +223,8 @@ struct tls_context {
unsigned long flags; unsigned long flags;
bool in_tcp_sendpages; bool in_tcp_sendpages;
bool pending_open_record_frags;
u16 pending_open_record_frags;
int (*push_pending_record)(struct sock *sk, int flags); int (*push_pending_record)(struct sock *sk, int flags);
void (*sk_write_space)(struct sock *sk); void (*sk_write_space)(struct sock *sk);
......
...@@ -73,6 +73,45 @@ int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, ...@@ -73,6 +73,45 @@ int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
} }
EXPORT_SYMBOL_GPL(sk_msg_alloc); EXPORT_SYMBOL_GPL(sk_msg_alloc);
int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
u32 off, u32 len)
{
int i = src->sg.start;
struct scatterlist *sge = sk_msg_elem(src, i);
u32 sge_len, sge_off;
if (sk_msg_full(dst))
return -ENOSPC;
while (off) {
if (sge->length > off)
break;
off -= sge->length;
sk_msg_iter_var_next(i);
if (i == src->sg.end && off)
return -ENOSPC;
sge = sk_msg_elem(src, i);
}
while (len) {
sge_len = sge->length - off;
sge_off = sge->offset + off;
if (sge_len > len)
sge_len = len;
off = 0;
len -= sge_len;
sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off);
sk_mem_charge(sk, sge_len);
sk_msg_iter_var_next(i);
if (i == src->sg.end && len)
return -ENOSPC;
sge = sk_msg_elem(src, i);
}
return 0;
}
EXPORT_SYMBOL_GPL(sk_msg_clone);
void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes) void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes)
{ {
int i = msg->sg.start; int i = msg->sg.start;
......
...@@ -2238,67 +2238,6 @@ bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) ...@@ -2238,67 +2238,6 @@ bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
} }
EXPORT_SYMBOL(sk_page_frag_refill); EXPORT_SYMBOL(sk_page_frag_refill);
int sk_alloc_sg(struct sock *sk, int len, struct scatterlist *sg,
int sg_start, int *sg_curr_index, unsigned int *sg_curr_size,
int first_coalesce)
{
int sg_curr = *sg_curr_index, use = 0, rc = 0;
unsigned int size = *sg_curr_size;
struct page_frag *pfrag;
struct scatterlist *sge;
len -= size;
pfrag = sk_page_frag(sk);
while (len > 0) {
unsigned int orig_offset;
if (!sk_page_frag_refill(sk, pfrag)) {
rc = -ENOMEM;
goto out;
}
use = min_t(int, len, pfrag->size - pfrag->offset);
if (!sk_wmem_schedule(sk, use)) {
rc = -ENOMEM;
goto out;
}
sk_mem_charge(sk, use);
size += use;
orig_offset = pfrag->offset;
pfrag->offset += use;
sge = sg + sg_curr - 1;
if (sg_curr > first_coalesce && sg_page(sge) == pfrag->page &&
sge->offset + sge->length == orig_offset) {
sge->length += use;
} else {
sge = sg + sg_curr;
sg_unmark_end(sge);
sg_set_page(sge, pfrag->page, use, orig_offset);
get_page(pfrag->page);
sg_curr++;
if (sg_curr == MAX_SKB_FRAGS)
sg_curr = 0;
if (sg_curr == sg_start) {
rc = -ENOSPC;
break;
}
}
len -= use;
}
out:
*sg_curr_size = size;
*sg_curr_index = sg_curr;
return rc;
}
EXPORT_SYMBOL(sk_alloc_sg);
static void __lock_sock(struct sock *sk) static void __lock_sock(struct sock *sk)
__releases(&sk->sk_lock.slock) __releases(&sk->sk_lock.slock)
__acquires(&sk->sk_lock.slock) __acquires(&sk->sk_lock.slock)
......
...@@ -8,6 +8,7 @@ config TLS ...@@ -8,6 +8,7 @@ config TLS
select CRYPTO_AES select CRYPTO_AES
select CRYPTO_GCM select CRYPTO_GCM
select STREAM_PARSER select STREAM_PARSER
select NET_SOCK_MSG
default n default n
---help--- ---help---
Enable kernel support for TLS protocol. This allows symmetric Enable kernel support for TLS protocol. This allows symmetric
......
...@@ -421,7 +421,7 @@ static int tls_push_data(struct sock *sk, ...@@ -421,7 +421,7 @@ static int tls_push_data(struct sock *sk,
tls_push_record_flags = flags; tls_push_record_flags = flags;
if (more) { if (more) {
tls_ctx->pending_open_record_frags = tls_ctx->pending_open_record_frags =
record->num_frags; !!record->num_frags;
break; break;
} }
......
...@@ -213,153 +213,49 @@ static int tls_do_decryption(struct sock *sk, ...@@ -213,153 +213,49 @@ static int tls_do_decryption(struct sock *sk,
return ret; return ret;
} }
static void trim_sg(struct sock *sk, struct scatterlist *sg, static void tls_trim_both_msgs(struct sock *sk, int target_size)
int *sg_num_elem, unsigned int *sg_size, int target_size)
{
int i = *sg_num_elem - 1;
int trim = *sg_size - target_size;
if (trim <= 0) {
WARN_ON(trim < 0);
return;
}
*sg_size = target_size;
while (trim >= sg[i].length) {
trim -= sg[i].length;
sk_mem_uncharge(sk, sg[i].length);
put_page(sg_page(&sg[i]));
i--;
if (i < 0)
goto out;
}
sg[i].length -= trim;
sk_mem_uncharge(sk, trim);
out:
*sg_num_elem = i + 1;
}
static void trim_both_sgl(struct sock *sk, int target_size)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct tls_rec *rec = ctx->open_rec; struct tls_rec *rec = ctx->open_rec;
trim_sg(sk, &rec->sg_plaintext_data[1], sk_msg_trim(sk, &rec->msg_plaintext, target_size);
&rec->sg_plaintext_num_elem,
&rec->sg_plaintext_size,
target_size);
if (target_size > 0) if (target_size > 0)
target_size += tls_ctx->tx.overhead_size; target_size += tls_ctx->tx.overhead_size;
sk_msg_trim(sk, &rec->msg_encrypted, target_size);
trim_sg(sk, &rec->sg_encrypted_data[1],
&rec->sg_encrypted_num_elem,
&rec->sg_encrypted_size,
target_size);
} }
static int alloc_encrypted_sg(struct sock *sk, int len) static int tls_alloc_encrypted_msg(struct sock *sk, int len)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct tls_rec *rec = ctx->open_rec; struct tls_rec *rec = ctx->open_rec;
int rc = 0; struct sk_msg *msg_en = &rec->msg_encrypted;
rc = sk_alloc_sg(sk, len,
&rec->sg_encrypted_data[1], 0,
&rec->sg_encrypted_num_elem,
&rec->sg_encrypted_size, 0);
if (rc == -ENOSPC)
rec->sg_encrypted_num_elem =
ARRAY_SIZE(rec->sg_encrypted_data) - 1;
return rc; return sk_msg_alloc(sk, msg_en, len, 0);
} }
static int move_to_plaintext_sg(struct sock *sk, int required_size) static int tls_clone_plaintext_msg(struct sock *sk, int required)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct tls_rec *rec = ctx->open_rec; struct tls_rec *rec = ctx->open_rec;
struct scatterlist *plain_sg = &rec->sg_plaintext_data[1]; struct sk_msg *msg_pl = &rec->msg_plaintext;
struct scatterlist *enc_sg = &rec->sg_encrypted_data[1]; struct sk_msg *msg_en = &rec->msg_encrypted;
int enc_sg_idx = 0;
int skip, len; int skip, len;
if (rec->sg_plaintext_num_elem == MAX_SKB_FRAGS) /* We add page references worth len bytes from encrypted sg
return -ENOSPC; * at the end of plaintext sg. It is guaranteed that msg_en
/* We add page references worth len bytes from enc_sg at the
* end of plain_sg. It is guaranteed that sg_encrypted_data
* has enough required room (ensured by caller). * has enough required room (ensured by caller).
*/ */
len = required_size - rec->sg_plaintext_size; len = required - msg_pl->sg.size;
/* Skip initial bytes in sg_encrypted_data to be able /* Skip initial bytes in msg_en's data to be able to use
* to use same offset of both plain and encrypted data. * same offset of both plain and encrypted data.
*/ */
skip = tls_ctx->tx.prepend_size + rec->sg_plaintext_size; skip = tls_ctx->tx.prepend_size + msg_pl->sg.size;
while (enc_sg_idx < rec->sg_encrypted_num_elem) {
if (enc_sg[enc_sg_idx].length > skip)
break;
skip -= enc_sg[enc_sg_idx].length;
enc_sg_idx++;
}
/* unmark the end of plain_sg*/ return sk_msg_clone(sk, msg_pl, msg_en, skip, len);
sg_unmark_end(plain_sg + rec->sg_plaintext_num_elem - 1);
while (len) {
struct page *page = sg_page(&enc_sg[enc_sg_idx]);
int bytes = enc_sg[enc_sg_idx].length - skip;
int offset = enc_sg[enc_sg_idx].offset + skip;
if (bytes > len)
bytes = len;
else
enc_sg_idx++;
/* Skipping is required only one time */
skip = 0;
/* Increment page reference */
get_page(page);
sg_set_page(&plain_sg[rec->sg_plaintext_num_elem], page,
bytes, offset);
sk_mem_charge(sk, bytes);
len -= bytes;
rec->sg_plaintext_size += bytes;
rec->sg_plaintext_num_elem++;
if (rec->sg_plaintext_num_elem == MAX_SKB_FRAGS)
return -ENOSPC;
}
return 0;
}
static void free_sg(struct sock *sk, struct scatterlist *sg,
int *sg_num_elem, unsigned int *sg_size)
{
int i, n = *sg_num_elem;
for (i = 0; i < n; ++i) {
sk_mem_uncharge(sk, sg[i].length);
put_page(sg_page(&sg[i]));
}
*sg_num_elem = 0;
*sg_size = 0;
} }
static void tls_free_open_rec(struct sock *sk) static void tls_free_open_rec(struct sock *sk)
...@@ -372,14 +268,8 @@ static void tls_free_open_rec(struct sock *sk) ...@@ -372,14 +268,8 @@ static void tls_free_open_rec(struct sock *sk)
if (!rec) if (!rec)
return; return;
free_sg(sk, &rec->sg_encrypted_data[1], sk_msg_free(sk, &rec->msg_encrypted);
&rec->sg_encrypted_num_elem, sk_msg_free(sk, &rec->msg_plaintext);
&rec->sg_encrypted_size);
free_sg(sk, &rec->sg_plaintext_data[1],
&rec->sg_plaintext_num_elem,
&rec->sg_plaintext_size);
kfree(rec); kfree(rec);
} }
...@@ -388,6 +278,7 @@ int tls_tx_records(struct sock *sk, int flags) ...@@ -388,6 +278,7 @@ int tls_tx_records(struct sock *sk, int flags)
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct tls_rec *rec, *tmp; struct tls_rec *rec, *tmp;
struct sk_msg *msg_en;
int tx_flags, rc = 0; int tx_flags, rc = 0;
if (tls_is_partially_sent_record(tls_ctx)) { if (tls_is_partially_sent_record(tls_ctx)) {
...@@ -407,9 +298,7 @@ int tls_tx_records(struct sock *sk, int flags) ...@@ -407,9 +298,7 @@ int tls_tx_records(struct sock *sk, int flags)
* Remove the head of tx_list * Remove the head of tx_list
*/ */
list_del(&rec->list); list_del(&rec->list);
free_sg(sk, &rec->sg_plaintext_data[1], sk_msg_free(sk, &rec->msg_plaintext);
&rec->sg_plaintext_num_elem, &rec->sg_plaintext_size);
kfree(rec); kfree(rec);
} }
...@@ -421,17 +310,15 @@ int tls_tx_records(struct sock *sk, int flags) ...@@ -421,17 +310,15 @@ int tls_tx_records(struct sock *sk, int flags)
else else
tx_flags = flags; tx_flags = flags;
msg_en = &rec->msg_encrypted;
rc = tls_push_sg(sk, tls_ctx, rc = tls_push_sg(sk, tls_ctx,
&rec->sg_encrypted_data[1], &msg_en->sg.data[msg_en->sg.curr],
0, tx_flags); 0, tx_flags);
if (rc) if (rc)
goto tx_err; goto tx_err;
list_del(&rec->list); list_del(&rec->list);
free_sg(sk, &rec->sg_plaintext_data[1], sk_msg_free(sk, &rec->msg_plaintext);
&rec->sg_plaintext_num_elem,
&rec->sg_plaintext_size);
kfree(rec); kfree(rec);
} else { } else {
break; break;
...@@ -451,15 +338,18 @@ static void tls_encrypt_done(struct crypto_async_request *req, int err) ...@@ -451,15 +338,18 @@ static void tls_encrypt_done(struct crypto_async_request *req, int err)
struct sock *sk = req->data; struct sock *sk = req->data;
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct scatterlist *sge;
struct sk_msg *msg_en;
struct tls_rec *rec; struct tls_rec *rec;
bool ready = false; bool ready = false;
int pending; int pending;
rec = container_of(aead_req, struct tls_rec, aead_req); rec = container_of(aead_req, struct tls_rec, aead_req);
msg_en = &rec->msg_encrypted;
rec->sg_encrypted_data[1].offset -= tls_ctx->tx.prepend_size; sge = sk_msg_elem(msg_en, msg_en->sg.curr);
rec->sg_encrypted_data[1].length += tls_ctx->tx.prepend_size; sge->offset -= tls_ctx->tx.prepend_size;
sge->length += tls_ctx->tx.prepend_size;
/* Check if error is previously set on socket */ /* Check if error is previously set on socket */
if (err || sk->sk_err) { if (err || sk->sk_err) {
...@@ -497,31 +387,29 @@ static void tls_encrypt_done(struct crypto_async_request *req, int err) ...@@ -497,31 +387,29 @@ static void tls_encrypt_done(struct crypto_async_request *req, int err)
/* Schedule the transmission */ /* Schedule the transmission */
if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask))
schedule_delayed_work(&ctx->tx_work.work, 2); schedule_delayed_work(&ctx->tx_work.work, 1);
} }
static int tls_do_encryption(struct sock *sk, static int tls_do_encryption(struct sock *sk,
struct tls_context *tls_ctx, struct tls_context *tls_ctx,
struct tls_sw_context_tx *ctx, struct tls_sw_context_tx *ctx,
struct aead_request *aead_req, struct aead_request *aead_req,
size_t data_len) size_t data_len, u32 start)
{ {
struct tls_rec *rec = ctx->open_rec; struct tls_rec *rec = ctx->open_rec;
struct scatterlist *plain_sg = rec->sg_plaintext_data; struct sk_msg *msg_en = &rec->msg_encrypted;
struct scatterlist *enc_sg = rec->sg_encrypted_data; struct scatterlist *sge = sk_msg_elem(msg_en, start);
int rc; int rc;
/* Skip the first index as it contains AAD data */ sge->offset += tls_ctx->tx.prepend_size;
rec->sg_encrypted_data[1].offset += tls_ctx->tx.prepend_size; sge->length -= tls_ctx->tx.prepend_size;
rec->sg_encrypted_data[1].length -= tls_ctx->tx.prepend_size;
/* If it is inplace crypto, then pass same SG list as both src, dst */ msg_en->sg.curr = start;
if (rec->inplace_crypto)
plain_sg = enc_sg;
aead_request_set_tfm(aead_req, ctx->aead_send); aead_request_set_tfm(aead_req, ctx->aead_send);
aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE); aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
aead_request_set_crypt(aead_req, plain_sg, enc_sg, aead_request_set_crypt(aead_req, rec->sg_aead_in,
rec->sg_aead_out,
data_len, tls_ctx->tx.iv); data_len, tls_ctx->tx.iv);
aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
...@@ -534,8 +422,8 @@ static int tls_do_encryption(struct sock *sk, ...@@ -534,8 +422,8 @@ static int tls_do_encryption(struct sock *sk,
rc = crypto_aead_encrypt(aead_req); rc = crypto_aead_encrypt(aead_req);
if (!rc || rc != -EINPROGRESS) { if (!rc || rc != -EINPROGRESS) {
atomic_dec(&ctx->encrypt_pending); atomic_dec(&ctx->encrypt_pending);
rec->sg_encrypted_data[1].offset -= tls_ctx->tx.prepend_size; sge->offset -= tls_ctx->tx.prepend_size;
rec->sg_encrypted_data[1].length += tls_ctx->tx.prepend_size; sge->length += tls_ctx->tx.prepend_size;
} }
if (!rc) { if (!rc) {
...@@ -557,34 +445,49 @@ static int tls_push_record(struct sock *sk, int flags, ...@@ -557,34 +445,49 @@ static int tls_push_record(struct sock *sk, int flags,
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct tls_rec *rec = ctx->open_rec; struct tls_rec *rec = ctx->open_rec;
struct sk_msg *msg_pl, *msg_en;
struct aead_request *req; struct aead_request *req;
int rc; int rc;
u32 i;
if (!rec) if (!rec)
return 0; return 0;
msg_pl = &rec->msg_plaintext;
msg_en = &rec->msg_encrypted;
rec->tx_flags = flags; rec->tx_flags = flags;
req = &rec->aead_req; req = &rec->aead_req;
sg_mark_end(rec->sg_plaintext_data + rec->sg_plaintext_num_elem); i = msg_pl->sg.end;
sg_mark_end(rec->sg_encrypted_data + rec->sg_encrypted_num_elem); sk_msg_iter_var_prev(i);
sg_mark_end(sk_msg_elem(msg_pl, i));
tls_make_aad(rec->aad_space, rec->sg_plaintext_size, i = msg_pl->sg.start;
sg_chain(rec->sg_aead_in, 2, rec->inplace_crypto ?
&msg_en->sg.data[i] : &msg_pl->sg.data[i]);
i = msg_en->sg.end;
sk_msg_iter_var_prev(i);
sg_mark_end(sk_msg_elem(msg_en, i));
i = msg_en->sg.start;
sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]);
tls_make_aad(rec->aad_space, msg_pl->sg.size,
tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size, tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size,
record_type); record_type);
tls_fill_prepend(tls_ctx, tls_fill_prepend(tls_ctx,
page_address(sg_page(&rec->sg_encrypted_data[1])) + page_address(sg_page(&msg_en->sg.data[i])) +
rec->sg_encrypted_data[1].offset, msg_en->sg.data[i].offset, msg_pl->sg.size,
rec->sg_plaintext_size, record_type); record_type);
tls_ctx->pending_open_record_frags = 0;
rc = tls_do_encryption(sk, tls_ctx, ctx, req, rec->sg_plaintext_size); tls_ctx->pending_open_record_frags = false;
if (rc == -EINPROGRESS)
return -EINPROGRESS;
rc = tls_do_encryption(sk, tls_ctx, ctx, req, msg_pl->sg.size, i);
if (rc < 0) { if (rc < 0) {
if (rc != -EINPROGRESS)
tls_err_abort(sk, EBADMSG); tls_err_abort(sk, EBADMSG);
return rc; return rc;
} }
...@@ -597,104 +500,11 @@ static int tls_sw_push_pending_record(struct sock *sk, int flags) ...@@ -597,104 +500,11 @@ static int tls_sw_push_pending_record(struct sock *sk, int flags)
return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA); return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA);
} }
static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
int length, int *pages_used,
unsigned int *size_used,
struct scatterlist *to, int to_max_pages,
bool charge)
{
struct page *pages[MAX_SKB_FRAGS];
size_t offset;
ssize_t copied, use;
int i = 0;
unsigned int size = *size_used;
int num_elem = *pages_used;
int rc = 0;
int maxpages;
while (length > 0) {
i = 0;
maxpages = to_max_pages - num_elem;
if (maxpages == 0) {
rc = -EFAULT;
goto out;
}
copied = iov_iter_get_pages(from, pages,
length,
maxpages, &offset);
if (copied <= 0) {
rc = -EFAULT;
goto out;
}
iov_iter_advance(from, copied);
length -= copied;
size += copied;
while (copied) {
use = min_t(int, copied, PAGE_SIZE - offset);
sg_set_page(&to[num_elem],
pages[i], use, offset);
sg_unmark_end(&to[num_elem]);
if (charge)
sk_mem_charge(sk, use);
offset = 0;
copied -= use;
++i;
++num_elem;
}
}
/* Mark the end in the last sg entry if newly added */
if (num_elem > *pages_used)
sg_mark_end(&to[num_elem - 1]);
out:
if (rc)
iov_iter_revert(from, size - *size_used);
*size_used = size;
*pages_used = num_elem;
return rc;
}
static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
int bytes)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct tls_rec *rec = ctx->open_rec;
struct scatterlist *sg = &rec->sg_plaintext_data[1];
int copy, i, rc = 0;
for (i = tls_ctx->pending_open_record_frags;
i < rec->sg_plaintext_num_elem; ++i) {
copy = sg[i].length;
if (copy_from_iter(
page_address(sg_page(&sg[i])) + sg[i].offset,
copy, from) != copy) {
rc = -EFAULT;
goto out;
}
bytes -= copy;
++tls_ctx->pending_open_record_frags;
if (!bytes)
break;
}
out:
return rc;
}
static struct tls_rec *get_rec(struct sock *sk) static struct tls_rec *get_rec(struct sock *sk)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct sk_msg *msg_pl, *msg_en;
struct tls_rec *rec; struct tls_rec *rec;
int mem_size; int mem_size;
...@@ -708,15 +518,21 @@ static struct tls_rec *get_rec(struct sock *sk) ...@@ -708,15 +518,21 @@ static struct tls_rec *get_rec(struct sock *sk)
if (!rec) if (!rec)
return NULL; return NULL;
sg_init_table(&rec->sg_plaintext_data[0], msg_pl = &rec->msg_plaintext;
ARRAY_SIZE(rec->sg_plaintext_data)); msg_en = &rec->msg_encrypted;
sg_init_table(&rec->sg_encrypted_data[0],
ARRAY_SIZE(rec->sg_encrypted_data));
sg_set_buf(&rec->sg_plaintext_data[0], rec->aad_space, sk_msg_init(msg_pl);
sk_msg_init(msg_en);
sg_init_table(rec->sg_aead_in, 2);
sg_set_buf(&rec->sg_aead_in[0], rec->aad_space,
sizeof(rec->aad_space)); sizeof(rec->aad_space));
sg_set_buf(&rec->sg_encrypted_data[0], rec->aad_space, sg_unmark_end(&rec->sg_aead_in[1]);
sg_init_table(rec->sg_aead_out, 2);
sg_set_buf(&rec->sg_aead_out[0], rec->aad_space,
sizeof(rec->aad_space)); sizeof(rec->aad_space));
sg_unmark_end(&rec->sg_aead_out[1]);
ctx->open_rec = rec; ctx->open_rec = rec;
rec->inplace_crypto = 1; rec->inplace_crypto = 1;
...@@ -735,6 +551,7 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) ...@@ -735,6 +551,7 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
bool is_kvec = msg->msg_iter.type & ITER_KVEC; bool is_kvec = msg->msg_iter.type & ITER_KVEC;
bool eor = !(msg->msg_flags & MSG_MORE); bool eor = !(msg->msg_flags & MSG_MORE);
size_t try_to_copy, copied = 0; size_t try_to_copy, copied = 0;
struct sk_msg *msg_pl, *msg_en;
struct tls_rec *rec; struct tls_rec *rec;
int required_size; int required_size;
int num_async = 0; int num_async = 0;
...@@ -778,23 +595,26 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) ...@@ -778,23 +595,26 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
goto send_end; goto send_end;
} }
orig_size = rec->sg_plaintext_size; msg_pl = &rec->msg_plaintext;
msg_en = &rec->msg_encrypted;
orig_size = msg_pl->sg.size;
full_record = false; full_record = false;
try_to_copy = msg_data_left(msg); try_to_copy = msg_data_left(msg);
record_room = TLS_MAX_PAYLOAD_SIZE - rec->sg_plaintext_size; record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size;
if (try_to_copy >= record_room) { if (try_to_copy >= record_room) {
try_to_copy = record_room; try_to_copy = record_room;
full_record = true; full_record = true;
} }
required_size = rec->sg_plaintext_size + try_to_copy + required_size = msg_pl->sg.size + try_to_copy +
tls_ctx->tx.overhead_size; tls_ctx->tx.overhead_size;
if (!sk_stream_memory_free(sk)) if (!sk_stream_memory_free(sk))
goto wait_for_sndbuf; goto wait_for_sndbuf;
alloc_encrypted: alloc_encrypted:
ret = alloc_encrypted_sg(sk, required_size); ret = tls_alloc_encrypted_msg(sk, required_size);
if (ret) { if (ret) {
if (ret != -ENOSPC) if (ret != -ENOSPC)
goto wait_for_memory; goto wait_for_memory;
...@@ -803,17 +623,13 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) ...@@ -803,17 +623,13 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
* actually allocated. The difference is due * actually allocated. The difference is due
* to max sg elements limit * to max sg elements limit
*/ */
try_to_copy -= required_size - rec->sg_encrypted_size; try_to_copy -= required_size - msg_en->sg.size;
full_record = true; full_record = true;
} }
if (!is_kvec && (full_record || eor) && !async_capable) { if (!is_kvec && (full_record || eor) && !async_capable) {
ret = zerocopy_from_iter(sk, &msg->msg_iter, ret = sk_msg_zerocopy_from_iter(sk, &msg->msg_iter,
try_to_copy, &rec->sg_plaintext_num_elem, msg_pl, try_to_copy);
&rec->sg_plaintext_size,
&rec->sg_plaintext_data[1],
ARRAY_SIZE(rec->sg_plaintext_data) - 1,
true);
if (ret) if (ret)
goto fallback_to_reg_send; goto fallback_to_reg_send;
...@@ -831,15 +647,12 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) ...@@ -831,15 +647,12 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
continue; continue;
fallback_to_reg_send: fallback_to_reg_send:
trim_sg(sk, &rec->sg_plaintext_data[1], sk_msg_trim(sk, msg_pl, orig_size);
&rec->sg_plaintext_num_elem,
&rec->sg_plaintext_size,
orig_size);
} }
required_size = rec->sg_plaintext_size + try_to_copy; required_size = msg_pl->sg.size + try_to_copy;
ret = move_to_plaintext_sg(sk, required_size); ret = tls_clone_plaintext_msg(sk, required_size);
if (ret) { if (ret) {
if (ret != -ENOSPC) if (ret != -ENOSPC)
goto send_end; goto send_end;
...@@ -848,20 +661,21 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) ...@@ -848,20 +661,21 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
* actually allocated. The difference is due * actually allocated. The difference is due
* to max sg elements limit * to max sg elements limit
*/ */
try_to_copy -= required_size - rec->sg_plaintext_size; try_to_copy -= required_size - msg_pl->sg.size;
full_record = true; full_record = true;
sk_msg_trim(sk, msg_en, msg_pl->sg.size +
trim_sg(sk, &rec->sg_encrypted_data[1],
&rec->sg_encrypted_num_elem,
&rec->sg_encrypted_size,
rec->sg_plaintext_size +
tls_ctx->tx.overhead_size); tls_ctx->tx.overhead_size);
} }
ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy); ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_pl,
if (ret) try_to_copy);
if (ret < 0)
goto trim_sgl; goto trim_sgl;
/* Open records defined only if successfully copied, otherwise
* we would trim the sg but not reset the open record frags.
*/
tls_ctx->pending_open_record_frags = true;
copied += try_to_copy; copied += try_to_copy;
if (full_record || eor) { if (full_record || eor) {
ret = tls_push_record(sk, msg->msg_flags, record_type); ret = tls_push_record(sk, msg->msg_flags, record_type);
...@@ -881,11 +695,11 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) ...@@ -881,11 +695,11 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
ret = sk_stream_wait_memory(sk, &timeo); ret = sk_stream_wait_memory(sk, &timeo);
if (ret) { if (ret) {
trim_sgl: trim_sgl:
trim_both_sgl(sk, orig_size); tls_trim_both_msgs(sk, orig_size);
goto send_end; goto send_end;
} }
if (rec->sg_encrypted_size < required_size) if (msg_en->sg.size < required_size)
goto alloc_encrypted; goto alloc_encrypted;
} }
...@@ -929,7 +743,7 @@ int tls_sw_sendpage(struct sock *sk, struct page *page, ...@@ -929,7 +743,7 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
unsigned char record_type = TLS_RECORD_TYPE_DATA; unsigned char record_type = TLS_RECORD_TYPE_DATA;
size_t orig_size = size; size_t orig_size = size;
struct scatterlist *sg; struct sk_msg *msg_pl;
struct tls_rec *rec; struct tls_rec *rec;
int num_async = 0; int num_async = 0;
bool full_record; bool full_record;
...@@ -970,20 +784,23 @@ int tls_sw_sendpage(struct sock *sk, struct page *page, ...@@ -970,20 +784,23 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,
goto sendpage_end; goto sendpage_end;
} }
msg_pl = &rec->msg_plaintext;
full_record = false; full_record = false;
record_room = TLS_MAX_PAYLOAD_SIZE - rec->sg_plaintext_size; record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size;
copy = size; copy = size;
if (copy >= record_room) { if (copy >= record_room) {
copy = record_room; copy = record_room;
full_record = true; full_record = true;
} }
required_size = rec->sg_plaintext_size + copy +
required_size = msg_pl->sg.size + copy +
tls_ctx->tx.overhead_size; tls_ctx->tx.overhead_size;
if (!sk_stream_memory_free(sk)) if (!sk_stream_memory_free(sk))
goto wait_for_sndbuf; goto wait_for_sndbuf;
alloc_payload: alloc_payload:
ret = alloc_encrypted_sg(sk, required_size); ret = tls_alloc_encrypted_msg(sk, required_size);
if (ret) { if (ret) {
if (ret != -ENOSPC) if (ret != -ENOSPC)
goto wait_for_memory; goto wait_for_memory;
...@@ -992,26 +809,18 @@ int tls_sw_sendpage(struct sock *sk, struct page *page, ...@@ -992,26 +809,18 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,
* actually allocated. The difference is due * actually allocated. The difference is due
* to max sg elements limit * to max sg elements limit
*/ */
copy -= required_size - rec->sg_plaintext_size; copy -= required_size - msg_pl->sg.size;
full_record = true; full_record = true;
} }
get_page(page); sk_msg_page_add(msg_pl, page, copy, offset);
sg = &rec->sg_plaintext_data[1] + rec->sg_plaintext_num_elem;
sg_set_page(sg, page, copy, offset);
sg_unmark_end(sg);
rec->sg_plaintext_num_elem++;
sk_mem_charge(sk, copy); sk_mem_charge(sk, copy);
offset += copy; offset += copy;
size -= copy; size -= copy;
rec->sg_plaintext_size += copy;
tls_ctx->pending_open_record_frags = rec->sg_plaintext_num_elem;
if (full_record || eor || tls_ctx->pending_open_record_frags = true;
rec->sg_plaintext_num_elem == if (full_record || eor || sk_msg_full(msg_pl)) {
ARRAY_SIZE(rec->sg_plaintext_data) - 1) {
rec->inplace_crypto = 0; rec->inplace_crypto = 0;
ret = tls_push_record(sk, flags, record_type); ret = tls_push_record(sk, flags, record_type);
if (ret) { if (ret) {
...@@ -1027,7 +836,7 @@ int tls_sw_sendpage(struct sock *sk, struct page *page, ...@@ -1027,7 +836,7 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,
wait_for_memory: wait_for_memory:
ret = sk_stream_wait_memory(sk, &timeo); ret = sk_stream_wait_memory(sk, &timeo);
if (ret) { if (ret) {
trim_both_sgl(sk, rec->sg_plaintext_size); tls_trim_both_msgs(sk, msg_pl->sg.size);
goto sendpage_end; goto sendpage_end;
} }
...@@ -1092,6 +901,64 @@ static struct sk_buff *tls_wait_data(struct sock *sk, int flags, ...@@ -1092,6 +901,64 @@ static struct sk_buff *tls_wait_data(struct sock *sk, int flags,
return skb; return skb;
} }
static int tls_setup_from_iter(struct sock *sk, struct iov_iter *from,
int length, int *pages_used,
unsigned int *size_used,
struct scatterlist *to,
int to_max_pages)
{
int rc = 0, i = 0, num_elem = *pages_used, maxpages;
struct page *pages[MAX_SKB_FRAGS];
unsigned int size = *size_used;
ssize_t copied, use;
size_t offset;
while (length > 0) {
i = 0;
maxpages = to_max_pages - num_elem;
if (maxpages == 0) {
rc = -EFAULT;
goto out;
}
copied = iov_iter_get_pages(from, pages,
length,
maxpages, &offset);
if (copied <= 0) {
rc = -EFAULT;
goto out;
}
iov_iter_advance(from, copied);
length -= copied;
size += copied;
while (copied) {
use = min_t(int, copied, PAGE_SIZE - offset);
sg_set_page(&to[num_elem],
pages[i], use, offset);
sg_unmark_end(&to[num_elem]);
/* We do not uncharge memory from this API */
offset = 0;
copied -= use;
i++;
num_elem++;
}
}
/* Mark the end in the last sg entry if newly added */
if (num_elem > *pages_used)
sg_mark_end(&to[num_elem - 1]);
out:
if (rc)
iov_iter_revert(from, size - *size_used);
*size_used = size;
*pages_used = num_elem;
return rc;
}
/* This function decrypts the input skb into either out_iov or in out_sg /* This function decrypts the input skb into either out_iov or in out_sg
* or in skb buffers itself. The input parameter 'zc' indicates if * or in skb buffers itself. The input parameter 'zc' indicates if
* zero-copy mode needs to be tried or not. With zero-copy mode, either * zero-copy mode needs to be tried or not. With zero-copy mode, either
...@@ -1189,9 +1056,9 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb, ...@@ -1189,9 +1056,9 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb,
sg_set_buf(&sgout[0], aad, TLS_AAD_SPACE_SIZE); sg_set_buf(&sgout[0], aad, TLS_AAD_SPACE_SIZE);
*chunk = 0; *chunk = 0;
err = zerocopy_from_iter(sk, out_iov, data_len, &pages, err = tls_setup_from_iter(sk, out_iov, data_len,
chunk, &sgout[1], &pages, chunk, &sgout[1],
(n_sgout - 1), false); (n_sgout - 1));
if (err < 0) if (err < 0)
goto fallback_to_reg_recv; goto fallback_to_reg_recv;
} else if (out_sg) { } else if (out_sg) {
...@@ -1619,25 +1486,15 @@ void tls_sw_free_resources_tx(struct sock *sk) ...@@ -1619,25 +1486,15 @@ void tls_sw_free_resources_tx(struct sock *sk)
rec = list_first_entry(&ctx->tx_list, rec = list_first_entry(&ctx->tx_list,
struct tls_rec, list); struct tls_rec, list);
free_sg(sk, &rec->sg_plaintext_data[1],
&rec->sg_plaintext_num_elem,
&rec->sg_plaintext_size);
list_del(&rec->list); list_del(&rec->list);
sk_msg_free(sk, &rec->msg_plaintext);
kfree(rec); kfree(rec);
} }
list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) { list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) {
free_sg(sk, &rec->sg_encrypted_data[1],
&rec->sg_encrypted_num_elem,
&rec->sg_encrypted_size);
free_sg(sk, &rec->sg_plaintext_data[1],
&rec->sg_plaintext_num_elem,
&rec->sg_plaintext_size);
list_del(&rec->list); list_del(&rec->list);
sk_msg_free(sk, &rec->msg_encrypted);
sk_msg_free(sk, &rec->msg_plaintext);
kfree(rec); kfree(rec);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment