Commit 4799ac81 authored by Boris Pismenny's avatar Boris Pismenny Committed by David S. Miller

tls: Add rx inline crypto offload

This patch completes the generic infrastructure to offload TLS crypto to a
network device. It enables the kernel to skip decryption and
authentication of some skbs marked as decrypted by the NIC. In the fast
path, all packets received are decrypted by the NIC and the performance
is comparable to plain TCP.

This infrastructure doesn't require a TCP offload engine. Instead, the
NIC only decrypts packets that contain the expected TCP sequence number.
Out-Of-Order TCP packets are provided unmodified. As a result, at the
worst case a received TLS record consists of both plaintext and ciphertext
packets. These partially decrypted records must be reencrypted,
only to be decrypted.

The notable differences between SW KTLS Rx and this offload are as
follows:
1. Partial decryption - Software must handle the case of a TLS record
that was only partially decrypted by HW. This can happen due to packet
reordering.
2. Resynchronization - tls_read_size calls the device driver to
resynchronize HW after HW lost track of TLS record framing in
the TCP stream.
Signed-off-by: default avatarBoris Pismenny <borisp@mellanox.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent b190a587
...@@ -83,6 +83,16 @@ struct tls_device { ...@@ -83,6 +83,16 @@ struct tls_device {
void (*unhash)(struct tls_device *device, struct sock *sk); void (*unhash)(struct tls_device *device, struct sock *sk);
}; };
enum {
TLS_BASE,
TLS_SW,
#ifdef CONFIG_TLS_DEVICE
TLS_HW,
#endif
TLS_HW_RECORD,
TLS_NUM_CONFIG,
};
struct tls_sw_context_tx { struct tls_sw_context_tx {
struct crypto_aead *aead_send; struct crypto_aead *aead_send;
struct crypto_wait async_wait; struct crypto_wait async_wait;
...@@ -197,6 +207,7 @@ struct tls_context { ...@@ -197,6 +207,7 @@ struct tls_context {
int (*push_pending_record)(struct sock *sk, int flags); int (*push_pending_record)(struct sock *sk, int flags);
void (*sk_write_space)(struct sock *sk); void (*sk_write_space)(struct sock *sk);
void (*sk_destruct)(struct sock *sk);
void (*sk_proto_close)(struct sock *sk, long timeout); void (*sk_proto_close)(struct sock *sk, long timeout);
int (*setsockopt)(struct sock *sk, int level, int (*setsockopt)(struct sock *sk, int level,
...@@ -209,13 +220,27 @@ struct tls_context { ...@@ -209,13 +220,27 @@ struct tls_context {
void (*unhash)(struct sock *sk); void (*unhash)(struct sock *sk);
}; };
struct tls_offload_context_rx {
/* sw must be the first member of tls_offload_context_rx */
struct tls_sw_context_rx sw;
atomic64_t resync_req;
u8 driver_state[];
/* The TLS layer reserves room for driver specific state
* Currently the belief is that there is not enough
* driver specific state to justify another layer of indirection
*/
};
#define TLS_OFFLOAD_CONTEXT_SIZE_RX \
(ALIGN(sizeof(struct tls_offload_context_rx), sizeof(void *)) + \
TLS_DRIVER_STATE_SIZE)
int wait_on_pending_writer(struct sock *sk, long *timeo); int wait_on_pending_writer(struct sock *sk, long *timeo);
int tls_sk_query(struct sock *sk, int optname, char __user *optval, int tls_sk_query(struct sock *sk, int optname, char __user *optval,
int __user *optlen); int __user *optlen);
int tls_sk_attach(struct sock *sk, int optname, char __user *optval, int tls_sk_attach(struct sock *sk, int optname, char __user *optval,
unsigned int optlen); unsigned int optlen);
int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx); int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx);
int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
int tls_sw_sendpage(struct sock *sk, struct page *page, int tls_sw_sendpage(struct sock *sk, struct page *page,
...@@ -290,11 +315,19 @@ static inline bool tls_is_pending_open_record(struct tls_context *tls_ctx) ...@@ -290,11 +315,19 @@ static inline bool tls_is_pending_open_record(struct tls_context *tls_ctx)
return tls_ctx->pending_open_record_frags; return tls_ctx->pending_open_record_frags;
} }
struct sk_buff *
tls_validate_xmit_skb(struct sock *sk, struct net_device *dev,
struct sk_buff *skb);
static inline bool tls_is_sk_tx_device_offloaded(struct sock *sk) static inline bool tls_is_sk_tx_device_offloaded(struct sock *sk)
{ {
return sk_fullsock(sk) && #ifdef CONFIG_SOCK_VALIDATE_XMIT
/* matches smp_store_release in tls_set_device_offload */ return sk_fullsock(sk) &
smp_load_acquire(&sk->sk_destruct) == &tls_device_sk_destruct; (smp_load_acquire(&sk->sk_validate_xmit_skb) ==
&tls_validate_xmit_skb);
#else
return false;
#endif
} }
static inline void tls_err_abort(struct sock *sk, int err) static inline void tls_err_abort(struct sock *sk, int err)
...@@ -387,10 +420,27 @@ tls_offload_ctx_tx(const struct tls_context *tls_ctx) ...@@ -387,10 +420,27 @@ tls_offload_ctx_tx(const struct tls_context *tls_ctx)
return (struct tls_offload_context_tx *)tls_ctx->priv_ctx_tx; return (struct tls_offload_context_tx *)tls_ctx->priv_ctx_tx;
} }
static inline struct tls_offload_context_rx *
tls_offload_ctx_rx(const struct tls_context *tls_ctx)
{
return (struct tls_offload_context_rx *)tls_ctx->priv_ctx_rx;
}
/* The TLS context is valid until sk_destruct is called */
static inline void tls_offload_rx_resync_request(struct sock *sk, __be32 seq)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx);
atomic64_set(&rx_ctx->resync_req, ((((uint64_t)seq) << 32) | 1));
}
int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg, int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
unsigned char *record_type); unsigned char *record_type);
void tls_register_device(struct tls_device *device); void tls_register_device(struct tls_device *device);
void tls_unregister_device(struct tls_device *device); void tls_unregister_device(struct tls_device *device);
int tls_device_decrypted(struct sock *sk, struct sk_buff *skb);
int decrypt_skb(struct sock *sk, struct sk_buff *skb, int decrypt_skb(struct sock *sk, struct sk_buff *skb,
struct scatterlist *sgout); struct scatterlist *sgout);
...@@ -402,4 +452,9 @@ int tls_sw_fallback_init(struct sock *sk, ...@@ -402,4 +452,9 @@ int tls_sw_fallback_init(struct sock *sk,
struct tls_offload_context_tx *offload_ctx, struct tls_offload_context_tx *offload_ctx,
struct tls_crypto_info *crypto_info); struct tls_crypto_info *crypto_info);
int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx);
void tls_device_offload_cleanup_rx(struct sock *sk);
void handle_device_resync(struct sock *sk, u32 seq, u64 rcd_sn);
#endif /* _TLS_OFFLOAD_H */ #endif /* _TLS_OFFLOAD_H */
This diff is collapsed.
...@@ -413,6 +413,7 @@ struct sk_buff *tls_validate_xmit_skb(struct sock *sk, ...@@ -413,6 +413,7 @@ struct sk_buff *tls_validate_xmit_skb(struct sock *sk,
return tls_sw_fallback(sk, skb); return tls_sw_fallback(sk, skb);
} }
EXPORT_SYMBOL_GPL(tls_validate_xmit_skb);
int tls_sw_fallback_init(struct sock *sk, int tls_sw_fallback_init(struct sock *sk,
struct tls_offload_context_tx *offload_ctx, struct tls_offload_context_tx *offload_ctx,
......
...@@ -51,15 +51,6 @@ enum { ...@@ -51,15 +51,6 @@ enum {
TLSV6, TLSV6,
TLS_NUM_PROTS, TLS_NUM_PROTS,
}; };
enum {
TLS_BASE,
TLS_SW,
#ifdef CONFIG_TLS_DEVICE
TLS_HW,
#endif
TLS_HW_RECORD,
TLS_NUM_CONFIG,
};
static struct proto *saved_tcpv6_prot; static struct proto *saved_tcpv6_prot;
static DEFINE_MUTEX(tcpv6_prot_mutex); static DEFINE_MUTEX(tcpv6_prot_mutex);
...@@ -290,7 +281,10 @@ static void tls_sk_proto_close(struct sock *sk, long timeout) ...@@ -290,7 +281,10 @@ static void tls_sk_proto_close(struct sock *sk, long timeout)
} }
#ifdef CONFIG_TLS_DEVICE #ifdef CONFIG_TLS_DEVICE
if (ctx->tx_conf != TLS_HW) { if (ctx->rx_conf == TLS_HW)
tls_device_offload_cleanup_rx(sk);
if (ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW) {
#else #else
{ {
#endif #endif
...@@ -470,9 +464,17 @@ static int do_tls_setsockopt_conf(struct sock *sk, char __user *optval, ...@@ -470,9 +464,17 @@ static int do_tls_setsockopt_conf(struct sock *sk, char __user *optval,
conf = TLS_SW; conf = TLS_SW;
} }
} else { } else {
#ifdef CONFIG_TLS_DEVICE
rc = tls_set_device_offload_rx(sk, ctx);
conf = TLS_HW;
if (rc) {
#else
{
#endif
rc = tls_set_sw_offload(sk, ctx, 0); rc = tls_set_sw_offload(sk, ctx, 0);
conf = TLS_SW; conf = TLS_SW;
} }
}
if (rc) if (rc)
goto err_crypto_info; goto err_crypto_info;
...@@ -629,6 +631,12 @@ static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], ...@@ -629,6 +631,12 @@ static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW]; prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW];
prot[TLS_HW][TLS_SW].sendmsg = tls_device_sendmsg; prot[TLS_HW][TLS_SW].sendmsg = tls_device_sendmsg;
prot[TLS_HW][TLS_SW].sendpage = tls_device_sendpage; prot[TLS_HW][TLS_SW].sendpage = tls_device_sendpage;
prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW];
prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW];
prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW];
#endif #endif
prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base; prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
......
...@@ -654,16 +654,25 @@ static struct sk_buff *tls_wait_data(struct sock *sk, int flags, ...@@ -654,16 +654,25 @@ static struct sk_buff *tls_wait_data(struct sock *sk, int flags,
} }
static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb,
struct scatterlist *sgout) struct scatterlist *sgout, bool *zc)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
struct strp_msg *rxm = strp_msg(skb); struct strp_msg *rxm = strp_msg(skb);
int err = 0; int err = 0;
#ifdef CONFIG_TLS_DEVICE
err = tls_device_decrypted(sk, skb);
if (err < 0)
return err;
#endif
if (!ctx->decrypted) {
err = decrypt_skb(sk, skb, sgout); err = decrypt_skb(sk, skb, sgout);
if (err < 0) if (err < 0)
return err; return err;
} else {
*zc = false;
}
rxm->offset += tls_ctx->rx.prepend_size; rxm->offset += tls_ctx->rx.prepend_size;
rxm->full_len -= tls_ctx->rx.overhead_size; rxm->full_len -= tls_ctx->rx.overhead_size;
...@@ -820,7 +829,7 @@ int tls_sw_recvmsg(struct sock *sk, ...@@ -820,7 +829,7 @@ int tls_sw_recvmsg(struct sock *sk,
if (err < 0) if (err < 0)
goto fallback_to_reg_recv; goto fallback_to_reg_recv;
err = decrypt_skb_update(sk, skb, sgin); err = decrypt_skb_update(sk, skb, sgin, &zc);
for (; pages > 0; pages--) for (; pages > 0; pages--)
put_page(sg_page(&sgin[pages])); put_page(sg_page(&sgin[pages]));
if (err < 0) { if (err < 0) {
...@@ -829,7 +838,7 @@ int tls_sw_recvmsg(struct sock *sk, ...@@ -829,7 +838,7 @@ int tls_sw_recvmsg(struct sock *sk,
} }
} else { } else {
fallback_to_reg_recv: fallback_to_reg_recv:
err = decrypt_skb_update(sk, skb, NULL); err = decrypt_skb_update(sk, skb, NULL, &zc);
if (err < 0) { if (err < 0) {
tls_err_abort(sk, EBADMSG); tls_err_abort(sk, EBADMSG);
goto recv_end; goto recv_end;
...@@ -884,6 +893,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, ...@@ -884,6 +893,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
int err = 0; int err = 0;
long timeo; long timeo;
int chunk; int chunk;
bool zc;
lock_sock(sk); lock_sock(sk);
...@@ -900,7 +910,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, ...@@ -900,7 +910,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
} }
if (!ctx->decrypted) { if (!ctx->decrypted) {
err = decrypt_skb_update(sk, skb, NULL); err = decrypt_skb_update(sk, skb, NULL, &zc);
if (err < 0) { if (err < 0) {
tls_err_abort(sk, EBADMSG); tls_err_abort(sk, EBADMSG);
...@@ -989,6 +999,10 @@ static int tls_read_size(struct strparser *strp, struct sk_buff *skb) ...@@ -989,6 +999,10 @@ static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
goto read_failure; goto read_failure;
} }
#ifdef CONFIG_TLS_DEVICE
handle_device_resync(strp->sk, TCP_SKB_CB(skb)->seq + rxm->offset,
*(u64*)tls_ctx->rx.rec_seq);
#endif
return data_len + TLS_HEADER_SIZE; return data_len + TLS_HEADER_SIZE;
read_failure: read_failure:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment