Commit 47ebcc0b authored by Yossi Kuperman's avatar Yossi Kuperman Committed by Steffen Klassert

xfrm: Add support for network devices capable of removing the ESP trailer

In conjunction with crypto offload [1], removing the ESP trailer by
hardware can potentially improve the performance by avoiding (1) a
cache miss incurred by reading the nexthdr field and (2) the necessity
to calculate the csum value of the trailer in order to keep skb->csum
valid.

This patch introduces the changes to the xfrm stack and merely serves
as an infrastructure. Subsequent patch to mlx5 driver will put this to
a good use.

[1] https://www.mail-archive.com/netdev@vger.kernel.org/msg175733.htmlSigned-off-by: default avatarYossi Kuperman <yossiku@mellanox.com>
Signed-off-by: default avatarSteffen Klassert <steffen.klassert@secunet.com>
parent acfb98b9
...@@ -1019,6 +1019,7 @@ struct xfrm_offload { ...@@ -1019,6 +1019,7 @@ struct xfrm_offload {
#define CRYPTO_FALLBACK 8 #define CRYPTO_FALLBACK 8
#define XFRM_GSO_SEGMENT 16 #define XFRM_GSO_SEGMENT 16
#define XFRM_GRO 32 #define XFRM_GRO 32
#define XFRM_ESP_NO_TRAILER 64
__u32 status; __u32 status;
#define CRYPTO_SUCCESS 1 #define CRYPTO_SUCCESS 1
......
...@@ -499,19 +499,59 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb) ...@@ -499,19 +499,59 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb)
return esp_output_tail(x, skb, &esp); return esp_output_tail(x, skb, &esp);
} }
static inline int esp_remove_trailer(struct sk_buff *skb)
{
struct xfrm_state *x = xfrm_input_state(skb);
struct xfrm_offload *xo = xfrm_offload(skb);
struct crypto_aead *aead = x->data;
int alen, hlen, elen;
int padlen, trimlen;
__wsum csumdiff;
u8 nexthdr[2];
int ret;
alen = crypto_aead_authsize(aead);
hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
elen = skb->len - hlen;
if (xo && (xo->flags & XFRM_ESP_NO_TRAILER)) {
ret = xo->proto;
goto out;
}
if (skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2))
BUG();
ret = -EINVAL;
padlen = nexthdr[0];
if (padlen + 2 + alen >= elen) {
net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",
padlen + 2, elen - alen);
goto out;
}
trimlen = alen + padlen + 2;
if (skb->ip_summed == CHECKSUM_COMPLETE) {
csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);
skb->csum = csum_block_sub(skb->csum, csumdiff,
skb->len - trimlen);
}
pskb_trim(skb, skb->len - trimlen);
ret = nexthdr[1];
out:
return ret;
}
int esp_input_done2(struct sk_buff *skb, int err) int esp_input_done2(struct sk_buff *skb, int err)
{ {
const struct iphdr *iph; const struct iphdr *iph;
struct xfrm_state *x = xfrm_input_state(skb); struct xfrm_state *x = xfrm_input_state(skb);
struct xfrm_offload *xo = xfrm_offload(skb); struct xfrm_offload *xo = xfrm_offload(skb);
struct crypto_aead *aead = x->data; struct crypto_aead *aead = x->data;
int alen = crypto_aead_authsize(aead);
int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead); int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
int elen = skb->len - hlen;
int ihl; int ihl;
u8 nexthdr[2];
int padlen, trimlen;
__wsum csumdiff;
if (!xo || (xo && !(xo->flags & CRYPTO_DONE))) if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))
kfree(ESP_SKB_CB(skb)->tmp); kfree(ESP_SKB_CB(skb)->tmp);
...@@ -519,16 +559,10 @@ int esp_input_done2(struct sk_buff *skb, int err) ...@@ -519,16 +559,10 @@ int esp_input_done2(struct sk_buff *skb, int err)
if (unlikely(err)) if (unlikely(err))
goto out; goto out;
if (skb_copy_bits(skb, skb->len-alen-2, nexthdr, 2)) err = esp_remove_trailer(skb);
BUG(); if (unlikely(err < 0))
err = -EINVAL;
padlen = nexthdr[0];
if (padlen + 2 + alen >= elen)
goto out; goto out;
/* ... check padding bits here. Silly. :-) */
iph = ip_hdr(skb); iph = ip_hdr(skb);
ihl = iph->ihl * 4; ihl = iph->ihl * 4;
...@@ -569,22 +603,12 @@ int esp_input_done2(struct sk_buff *skb, int err) ...@@ -569,22 +603,12 @@ int esp_input_done2(struct sk_buff *skb, int err)
skb->ip_summed = CHECKSUM_UNNECESSARY; skb->ip_summed = CHECKSUM_UNNECESSARY;
} }
trimlen = alen + padlen + 2;
if (skb->ip_summed == CHECKSUM_COMPLETE) {
csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);
skb->csum = csum_block_sub(skb->csum, csumdiff,
skb->len - trimlen);
}
pskb_trim(skb, skb->len - trimlen);
skb_pull_rcsum(skb, hlen); skb_pull_rcsum(skb, hlen);
if (x->props.mode == XFRM_MODE_TUNNEL) if (x->props.mode == XFRM_MODE_TUNNEL)
skb_reset_transport_header(skb); skb_reset_transport_header(skb);
else else
skb_set_transport_header(skb, -ihl); skb_set_transport_header(skb, -ihl);
err = nexthdr[1];
/* RFC4303: Drop dummy packets without any error */ /* RFC4303: Drop dummy packets without any error */
if (err == IPPROTO_NONE) if (err == IPPROTO_NONE)
err = -EINVAL; err = -EINVAL;
......
...@@ -461,29 +461,30 @@ static int esp6_output(struct xfrm_state *x, struct sk_buff *skb) ...@@ -461,29 +461,30 @@ static int esp6_output(struct xfrm_state *x, struct sk_buff *skb)
return esp6_output_tail(x, skb, &esp); return esp6_output_tail(x, skb, &esp);
} }
int esp6_input_done2(struct sk_buff *skb, int err) static inline int esp_remove_trailer(struct sk_buff *skb)
{ {
struct xfrm_state *x = xfrm_input_state(skb); struct xfrm_state *x = xfrm_input_state(skb);
struct xfrm_offload *xo = xfrm_offload(skb); struct xfrm_offload *xo = xfrm_offload(skb);
struct crypto_aead *aead = x->data; struct crypto_aead *aead = x->data;
int alen = crypto_aead_authsize(aead); int alen, hlen, elen;
int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
int elen = skb->len - hlen;
int hdr_len = skb_network_header_len(skb);
int padlen, trimlen; int padlen, trimlen;
__wsum csumdiff; __wsum csumdiff;
u8 nexthdr[2]; u8 nexthdr[2];
int ret;
if (!xo || (xo && !(xo->flags & CRYPTO_DONE))) alen = crypto_aead_authsize(aead);
kfree(ESP_SKB_CB(skb)->tmp); hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
elen = skb->len - hlen;
if (unlikely(err)) if (xo && (xo->flags & XFRM_ESP_NO_TRAILER)) {
ret = xo->proto;
goto out; goto out;
}
if (skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2)) if (skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2))
BUG(); BUG();
err = -EINVAL; ret = -EINVAL;
padlen = nexthdr[0]; padlen = nexthdr[0];
if (padlen + 2 + alen >= elen) { if (padlen + 2 + alen >= elen) {
net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n", net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",
...@@ -491,26 +492,46 @@ int esp6_input_done2(struct sk_buff *skb, int err) ...@@ -491,26 +492,46 @@ int esp6_input_done2(struct sk_buff *skb, int err)
goto out; goto out;
} }
/* ... check padding bits here. Silly. :-) */
trimlen = alen + padlen + 2; trimlen = alen + padlen + 2;
if (skb->ip_summed == CHECKSUM_COMPLETE) { if (skb->ip_summed == CHECKSUM_COMPLETE) {
skb_postpull_rcsum(skb, skb_network_header(skb),
skb_network_header_len(skb));
csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0); csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);
skb->csum = csum_block_sub(skb->csum, csumdiff, skb->csum = csum_block_sub(skb->csum, csumdiff,
skb->len - trimlen); skb->len - trimlen);
} }
pskb_trim(skb, skb->len - trimlen); pskb_trim(skb, skb->len - trimlen);
ret = nexthdr[1];
out:
return ret;
}
int esp6_input_done2(struct sk_buff *skb, int err)
{
struct xfrm_state *x = xfrm_input_state(skb);
struct xfrm_offload *xo = xfrm_offload(skb);
struct crypto_aead *aead = x->data;
int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
int hdr_len = skb_network_header_len(skb);
if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))
kfree(ESP_SKB_CB(skb)->tmp);
if (unlikely(err))
goto out;
err = esp_remove_trailer(skb);
if (unlikely(err < 0))
goto out;
skb_postpull_rcsum(skb, skb_network_header(skb),
skb_network_header_len(skb));
skb_pull_rcsum(skb, hlen); skb_pull_rcsum(skb, hlen);
if (x->props.mode == XFRM_MODE_TUNNEL) if (x->props.mode == XFRM_MODE_TUNNEL)
skb_reset_transport_header(skb); skb_reset_transport_header(skb);
else else
skb_set_transport_header(skb, -hdr_len); skb_set_transport_header(skb, -hdr_len);
err = nexthdr[1];
/* RFC4303: Drop dummy packets without any error */ /* RFC4303: Drop dummy packets without any error */
if (err == IPPROTO_NONE) if (err == IPPROTO_NONE)
err = -EINVAL; err = -EINVAL;
......
...@@ -247,6 +247,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) ...@@ -247,6 +247,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
goto drop; goto drop;
} }
if (xo->status & CRYPTO_INVALID_PROTOCOL) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
goto drop;
}
XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
goto drop; goto drop;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment