Commit d77e38e6 authored by Steffen Klassert's avatar Steffen Klassert

xfrm: Add an IPsec hardware offloading API

This patch adds all the bits that are needed to do
IPsec hardware offload for IPsec states and ESP packets.
We add xfrmdev_ops to the net_device. xfrmdev_ops has
function pointers that are needed to manage the xfrm
states in the hardware and to do a per packet
offloading decision.

Joint work with:
Ilan Tayari <ilant@mellanox.com>
Guy Shapiro <guysh@mellanox.com>
Yossi Kuperman <yossiku@mellanox.com>
Signed-off-by: default avatarGuy Shapiro <guysh@mellanox.com>
Signed-off-by: default avatarIlan Tayari <ilant@mellanox.com>
Signed-off-by: default avatarYossi Kuperman <yossiku@mellanox.com>
Signed-off-by: default avatarSteffen Klassert <steffen.klassert@secunet.com>
parent c35fe410
...@@ -824,6 +824,16 @@ struct netdev_xdp { ...@@ -824,6 +824,16 @@ struct netdev_xdp {
}; };
}; };
#ifdef CONFIG_XFRM_OFFLOAD
struct xfrmdev_ops {
int (*xdo_dev_state_add) (struct xfrm_state *x);
void (*xdo_dev_state_delete) (struct xfrm_state *x);
void (*xdo_dev_state_free) (struct xfrm_state *x);
bool (*xdo_dev_offload_ok) (struct sk_buff *skb,
struct xfrm_state *x);
};
#endif
/* /*
* This structure defines the management hooks for network devices. * This structure defines the management hooks for network devices.
* The following hooks can be defined; unless noted otherwise, they are * The following hooks can be defined; unless noted otherwise, they are
...@@ -1697,6 +1707,10 @@ struct net_device { ...@@ -1697,6 +1707,10 @@ struct net_device {
const struct ndisc_ops *ndisc_ops; const struct ndisc_ops *ndisc_ops;
#endif #endif
#ifdef CONFIG_XFRM
const struct xfrmdev_ops *xfrmdev_ops;
#endif
const struct header_ops *header_ops; const struct header_ops *header_ops;
unsigned int flags; unsigned int flags;
......
...@@ -120,6 +120,13 @@ struct xfrm_state_walk { ...@@ -120,6 +120,13 @@ struct xfrm_state_walk {
struct xfrm_address_filter *filter; struct xfrm_address_filter *filter;
}; };
struct xfrm_state_offload {
struct net_device *dev;
unsigned long offload_handle;
unsigned int num_exthdrs;
u8 flags;
};
/* Full description of state of transformer. */ /* Full description of state of transformer. */
struct xfrm_state { struct xfrm_state {
possible_net_t xs_net; possible_net_t xs_net;
...@@ -207,6 +214,8 @@ struct xfrm_state { ...@@ -207,6 +214,8 @@ struct xfrm_state {
struct xfrm_lifetime_cur curlft; struct xfrm_lifetime_cur curlft;
struct tasklet_hrtimer mtimer; struct tasklet_hrtimer mtimer;
struct xfrm_state_offload xso;
/* used to fix curlft->add_time when changing date */ /* used to fix curlft->add_time when changing date */
long saved_tmo; long saved_tmo;
...@@ -1453,7 +1462,6 @@ struct xfrm6_tunnel { ...@@ -1453,7 +1462,6 @@ struct xfrm6_tunnel {
void xfrm_init(void); void xfrm_init(void);
void xfrm4_init(void); void xfrm4_init(void);
int xfrm_state_init(struct net *net); int xfrm_state_init(struct net *net);
void xfrm_dev_init(void);
void xfrm_state_fini(struct net *net); void xfrm_state_fini(struct net *net);
void xfrm4_state_init(void); void xfrm4_state_init(void);
void xfrm4_protocol_init(void); void xfrm4_protocol_init(void);
...@@ -1559,6 +1567,7 @@ struct xfrmk_spdinfo { ...@@ -1559,6 +1567,7 @@ struct xfrmk_spdinfo {
struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq); struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq);
int xfrm_state_delete(struct xfrm_state *x); int xfrm_state_delete(struct xfrm_state *x);
int xfrm_state_flush(struct net *net, u8 proto, bool task_valid); int xfrm_state_flush(struct net *net, u8 proto, bool task_valid);
int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid);
void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si); void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si);
void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si); void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si);
u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq); u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq);
...@@ -1641,6 +1650,11 @@ static inline int xfrm4_udp_encap_rcv(struct sock *sk, struct sk_buff *skb) ...@@ -1641,6 +1650,11 @@ static inline int xfrm4_udp_encap_rcv(struct sock *sk, struct sk_buff *skb)
} }
#endif #endif
struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif,
const xfrm_address_t *saddr,
const xfrm_address_t *daddr,
int family);
struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp); struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp);
void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type); void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type);
...@@ -1846,6 +1860,55 @@ static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb) ...@@ -1846,6 +1860,55 @@ static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb)
} }
#endif #endif
#ifdef CONFIG_XFRM_OFFLOAD
void __net_init xfrm_dev_init(void);
int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
struct xfrm_user_offload *xuo);
bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
static inline void xfrm_dev_state_delete(struct xfrm_state *x)
{
struct xfrm_state_offload *xso = &x->xso;
if (xso->dev)
xso->dev->xfrmdev_ops->xdo_dev_state_delete(x);
}
static inline void xfrm_dev_state_free(struct xfrm_state *x)
{
struct xfrm_state_offload *xso = &x->xso;
struct net_device *dev = xso->dev;
if (dev && dev->xfrmdev_ops) {
dev->xfrmdev_ops->xdo_dev_state_free(x);
xso->dev = NULL;
dev_put(dev);
}
}
#else
static inline void __net_init xfrm_dev_init(void)
{
}
static inline int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo)
{
return 0;
}
static inline void xfrm_dev_state_delete(struct xfrm_state *x)
{
}
static inline void xfrm_dev_state_free(struct xfrm_state *x)
{
}
static inline bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
{
return false;
}
#endif
static inline int xfrm_mark_get(struct nlattr **attrs, struct xfrm_mark *m) static inline int xfrm_mark_get(struct nlattr **attrs, struct xfrm_mark *m)
{ {
if (attrs[XFRMA_MARK]) if (attrs[XFRMA_MARK])
......
...@@ -303,6 +303,7 @@ enum xfrm_attr_type_t { ...@@ -303,6 +303,7 @@ enum xfrm_attr_type_t {
XFRMA_PROTO, /* __u8 */ XFRMA_PROTO, /* __u8 */
XFRMA_ADDRESS_FILTER, /* struct xfrm_address_filter */ XFRMA_ADDRESS_FILTER, /* struct xfrm_address_filter */
XFRMA_PAD, XFRMA_PAD,
XFRMA_OFFLOAD_DEV, /* struct xfrm_state_offload */
__XFRMA_MAX __XFRMA_MAX
#define XFRMA_MAX (__XFRMA_MAX - 1) #define XFRMA_MAX (__XFRMA_MAX - 1)
...@@ -494,6 +495,13 @@ struct xfrm_address_filter { ...@@ -494,6 +495,13 @@ struct xfrm_address_filter {
__u8 dplen; __u8 dplen;
}; };
struct xfrm_user_offload {
int ifindex;
__u8 flags;
};
#define XFRM_OFFLOAD_IPV6 1
#define XFRM_OFFLOAD_INBOUND 2
#ifndef __KERNEL__ #ifndef __KERNEL__
/* backwards compatibility for userspace */ /* backwards compatibility for userspace */
#define XFRMGRP_ACQUIRE 1 #define XFRMGRP_ACQUIRE 1
......
...@@ -435,9 +435,6 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb) ...@@ -435,9 +435,6 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb)
aead_request_set_crypt(req, sg, dsg, ivlen + clen, iv); aead_request_set_crypt(req, sg, dsg, ivlen + clen, iv);
aead_request_set_ad(req, assoclen); aead_request_set_ad(req, assoclen);
seqno = cpu_to_be64(XFRM_SKB_CB(skb)->seq.output.low +
((u64)XFRM_SKB_CB(skb)->seq.output.hi << 32));
memset(iv, 0, ivlen); memset(iv, 0, ivlen);
memcpy(iv + ivlen - min(ivlen, 8), (u8 *)&seqno + 8 - min(ivlen, 8), memcpy(iv + ivlen - min(ivlen, 8), (u8 *)&seqno + 8 - min(ivlen, 8),
min(ivlen, 8)); min(ivlen, 8));
...@@ -470,6 +467,7 @@ static int esp_input_done2(struct sk_buff *skb, int err) ...@@ -470,6 +467,7 @@ static int esp_input_done2(struct sk_buff *skb, int err)
{ {
const struct iphdr *iph; const struct iphdr *iph;
struct xfrm_state *x = xfrm_input_state(skb); struct xfrm_state *x = xfrm_input_state(skb);
struct xfrm_offload *xo = xfrm_offload(skb);
struct crypto_aead *aead = x->data; struct crypto_aead *aead = x->data;
int alen = crypto_aead_authsize(aead); int alen = crypto_aead_authsize(aead);
int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead); int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
...@@ -478,6 +476,7 @@ static int esp_input_done2(struct sk_buff *skb, int err) ...@@ -478,6 +476,7 @@ static int esp_input_done2(struct sk_buff *skb, int err)
u8 nexthdr[2]; u8 nexthdr[2];
int padlen; int padlen;
if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))
kfree(ESP_SKB_CB(skb)->tmp); kfree(ESP_SKB_CB(skb)->tmp);
if (unlikely(err)) if (unlikely(err))
......
...@@ -29,7 +29,8 @@ static int xfrm4_tunnel_check_size(struct sk_buff *skb) ...@@ -29,7 +29,8 @@ static int xfrm4_tunnel_check_size(struct sk_buff *skb)
goto out; goto out;
mtu = dst_mtu(skb_dst(skb)); mtu = dst_mtu(skb_dst(skb));
if (skb->len > mtu) { if ((!skb_is_gso(skb) && skb->len > mtu) ||
(skb_is_gso(skb) && skb_gso_network_seglen(skb) > ip_skb_dst_mtu(skb->sk, skb))) {
skb->protocol = htons(ETH_P_IP); skb->protocol = htons(ETH_P_IP);
if (skb->sk) if (skb->sk)
......
...@@ -450,6 +450,7 @@ static int esp6_output(struct xfrm_state *x, struct sk_buff *skb) ...@@ -450,6 +450,7 @@ static int esp6_output(struct xfrm_state *x, struct sk_buff *skb)
static int esp_input_done2(struct sk_buff *skb, int err) static int esp_input_done2(struct sk_buff *skb, int err)
{ {
struct xfrm_state *x = xfrm_input_state(skb); struct xfrm_state *x = xfrm_input_state(skb);
struct xfrm_offload *xo = xfrm_offload(skb);
struct crypto_aead *aead = x->data; struct crypto_aead *aead = x->data;
int alen = crypto_aead_authsize(aead); int alen = crypto_aead_authsize(aead);
int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead); int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
...@@ -458,6 +459,7 @@ static int esp_input_done2(struct sk_buff *skb, int err) ...@@ -458,6 +459,7 @@ static int esp_input_done2(struct sk_buff *skb, int err)
int padlen; int padlen;
u8 nexthdr[2]; u8 nexthdr[2];
if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))
kfree(ESP_SKB_CB(skb)->tmp); kfree(ESP_SKB_CB(skb)->tmp);
if (unlikely(err)) if (unlikely(err))
......
...@@ -73,11 +73,16 @@ static int xfrm6_tunnel_check_size(struct sk_buff *skb) ...@@ -73,11 +73,16 @@ static int xfrm6_tunnel_check_size(struct sk_buff *skb)
int mtu, ret = 0; int mtu, ret = 0;
struct dst_entry *dst = skb_dst(skb); struct dst_entry *dst = skb_dst(skb);
if (skb->ignore_df)
goto out;
mtu = dst_mtu(dst); mtu = dst_mtu(dst);
if (mtu < IPV6_MIN_MTU) if (mtu < IPV6_MIN_MTU)
mtu = IPV6_MIN_MTU; mtu = IPV6_MIN_MTU;
if (!skb->ignore_df && skb->len > mtu) { if ((!skb_is_gso(skb) && skb->len > mtu) ||
(skb_is_gso(skb) &&
skb_gso_network_seglen(skb) > ip6_skb_dst_mtu(skb))) {
skb->dev = dst->dev; skb->dev = dst->dev;
skb->protocol = htons(ETH_P_IPV6); skb->protocol = htons(ETH_P_IPV6);
...@@ -89,7 +94,7 @@ static int xfrm6_tunnel_check_size(struct sk_buff *skb) ...@@ -89,7 +94,7 @@ static int xfrm6_tunnel_check_size(struct sk_buff *skb)
icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
ret = -EMSGSIZE; ret = -EMSGSIZE;
} }
out:
return ret; return ret;
} }
......
...@@ -4,7 +4,8 @@ ...@@ -4,7 +4,8 @@
obj-$(CONFIG_XFRM) := xfrm_policy.o xfrm_state.o xfrm_hash.o \ obj-$(CONFIG_XFRM) := xfrm_policy.o xfrm_state.o xfrm_hash.o \
xfrm_input.o xfrm_output.o \ xfrm_input.o xfrm_output.o \
xfrm_sysctl.o xfrm_replay.o xfrm_device.o xfrm_sysctl.o xfrm_replay.o
obj-$(CONFIG_XFRM_OFFLOAD) += xfrm_device.o
obj-$(CONFIG_XFRM_STATISTICS) += xfrm_proc.o obj-$(CONFIG_XFRM_STATISTICS) += xfrm_proc.o
obj-$(CONFIG_XFRM_ALGO) += xfrm_algo.o obj-$(CONFIG_XFRM_ALGO) += xfrm_algo.o
obj-$(CONFIG_XFRM_USER) += xfrm_user.o obj-$(CONFIG_XFRM_USER) += xfrm_user.o
......
...@@ -22,13 +22,149 @@ ...@@ -22,13 +22,149 @@
#include <net/xfrm.h> #include <net/xfrm.h>
#include <linux/notifier.h> #include <linux/notifier.h>
int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
struct xfrm_user_offload *xuo)
{
int err;
struct dst_entry *dst;
struct net_device *dev;
struct xfrm_state_offload *xso = &x->xso;
xfrm_address_t *saddr;
xfrm_address_t *daddr;
if (!x->type_offload)
return 0;
/* We don't yet support UDP encapsulation, TFC padding and ESN. */
if (x->encap || x->tfcpad || (x->props.flags & XFRM_STATE_ESN))
return 0;
dev = dev_get_by_index(net, xuo->ifindex);
if (!dev) {
if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
saddr = &x->props.saddr;
daddr = &x->id.daddr;
} else {
saddr = &x->id.daddr;
daddr = &x->props.saddr;
}
dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr, x->props.family);
if (IS_ERR(dst))
return 0;
dev = dst->dev;
dev_hold(dev);
dst_release(dst);
}
if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
dev_put(dev);
return 0;
}
xso->dev = dev;
xso->num_exthdrs = 1;
xso->flags = xuo->flags;
err = dev->xfrmdev_ops->xdo_dev_state_add(x);
if (err) {
dev_put(dev);
return err;
}
return 0;
}
EXPORT_SYMBOL_GPL(xfrm_dev_state_add);
bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
{
int mtu;
struct dst_entry *dst = skb_dst(skb);
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
struct net_device *dev = x->xso.dev;
if (!x->type_offload || x->encap)
return false;
if ((x->xso.offload_handle && (dev == dst->path->dev)) &&
!dst->child->xfrm && x->type->get_mtu) {
mtu = x->type->get_mtu(x, xdst->child_mtu_cached);
if (skb->len <= mtu)
goto ok;
if (skb_is_gso(skb) && skb_gso_validate_mtu(skb, mtu))
goto ok;
}
return false;
ok:
if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)
return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
return true;
}
EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);
int xfrm_dev_register(struct net_device *dev)
{
if ((dev->features & NETIF_F_HW_ESP) && !dev->xfrmdev_ops)
return NOTIFY_BAD;
if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
!(dev->features & NETIF_F_HW_ESP))
return NOTIFY_BAD;
return NOTIFY_DONE;
}
static int xfrm_dev_unregister(struct net_device *dev)
{
return NOTIFY_DONE;
}
static int xfrm_dev_feat_change(struct net_device *dev)
{
if ((dev->features & NETIF_F_HW_ESP) && !dev->xfrmdev_ops)
return NOTIFY_BAD;
else if (!(dev->features & NETIF_F_HW_ESP))
dev->xfrmdev_ops = NULL;
if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
!(dev->features & NETIF_F_HW_ESP))
return NOTIFY_BAD;
return NOTIFY_DONE;
}
static int xfrm_dev_down(struct net_device *dev)
{
if (dev->hw_features & NETIF_F_HW_ESP)
xfrm_dev_state_flush(dev_net(dev), dev, true);
xfrm_garbage_collect(dev_net(dev));
return NOTIFY_DONE;
}
static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr) static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
{ {
struct net_device *dev = netdev_notifier_info_to_dev(ptr); struct net_device *dev = netdev_notifier_info_to_dev(ptr);
switch (event) { switch (event) {
case NETDEV_REGISTER:
return xfrm_dev_register(dev);
case NETDEV_UNREGISTER:
return xfrm_dev_unregister(dev);
case NETDEV_FEAT_CHANGE:
return xfrm_dev_feat_change(dev);
case NETDEV_DOWN: case NETDEV_DOWN:
xfrm_garbage_collect(dev_net(dev)); return xfrm_dev_down(dev);
} }
return NOTIFY_DONE; return NOTIFY_DONE;
} }
......
...@@ -107,6 +107,8 @@ struct sec_path *secpath_dup(struct sec_path *src) ...@@ -107,6 +107,8 @@ struct sec_path *secpath_dup(struct sec_path *src)
sp->len = 0; sp->len = 0;
sp->olen = 0; sp->olen = 0;
memset(sp->ovec, 0, sizeof(sp->ovec[XFRM_MAX_OFFLOAD_DEPTH]));
if (src) { if (src) {
int i; int i;
...@@ -207,8 +209,9 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) ...@@ -207,8 +209,9 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
unsigned int family; unsigned int family;
int decaps = 0; int decaps = 0;
int async = 0; int async = 0;
struct xfrm_offload *xo;
bool xfrm_gro = false; bool xfrm_gro = false;
bool crypto_done = false;
struct xfrm_offload *xo = xfrm_offload(skb);
if (encap_type < 0) { if (encap_type < 0) {
x = xfrm_input_state(skb); x = xfrm_input_state(skb);
...@@ -226,6 +229,37 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) ...@@ -226,6 +229,37 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
goto lock; goto lock;
} }
if (xo && (xo->flags & CRYPTO_DONE)) {
crypto_done = true;
x = xfrm_input_state(skb);
family = XFRM_SPI_SKB_CB(skb)->family;
if (!(xo->status & CRYPTO_SUCCESS)) {
if (xo->status &
(CRYPTO_TRANSPORT_AH_AUTH_FAILED |
CRYPTO_TRANSPORT_ESP_AUTH_FAILED |
CRYPTO_TUNNEL_AH_AUTH_FAILED |
CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {
xfrm_audit_state_icvfail(x, skb,
x->type->proto);
x->stats.integrity_failed++;
XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
goto drop;
}
XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
goto drop;
}
if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
goto drop;
}
goto lock;
}
daddr = (xfrm_address_t *)(skb_network_header(skb) + daddr = (xfrm_address_t *)(skb_network_header(skb) +
XFRM_SPI_SKB_CB(skb)->daddroff); XFRM_SPI_SKB_CB(skb)->daddroff);
family = XFRM_SPI_SKB_CB(skb)->family; family = XFRM_SPI_SKB_CB(skb)->family;
...@@ -311,6 +345,9 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) ...@@ -311,6 +345,9 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
skb_dst_force(skb); skb_dst_force(skb);
dev_hold(skb->dev); dev_hold(skb->dev);
if (crypto_done)
nexthdr = x->type_offload->input_tail(x, skb);
else
nexthdr = x->type->input(x, skb); nexthdr = x->type->input(x, skb);
if (nexthdr == -EINPROGRESS) if (nexthdr == -EINPROGRESS)
......
...@@ -99,12 +99,13 @@ static int xfrm_output_one(struct sk_buff *skb, int err) ...@@ -99,12 +99,13 @@ static int xfrm_output_one(struct sk_buff *skb, int err)
skb_dst_force(skb); skb_dst_force(skb);
/* Inner headers are invalid now. */ if (xfrm_offload(skb)) {
skb->encapsulation = 0; x->type_offload->encap(x, skb);
} else {
err = x->type->output(x, skb); err = x->type->output(x, skb);
if (err == -EINPROGRESS) if (err == -EINPROGRESS)
goto out; goto out;
}
resume: resume:
if (err) { if (err) {
...@@ -200,8 +201,38 @@ static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb ...@@ -200,8 +201,38 @@ static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb
int xfrm_output(struct sock *sk, struct sk_buff *skb) int xfrm_output(struct sock *sk, struct sk_buff *skb)
{ {
struct net *net = dev_net(skb_dst(skb)->dev); struct net *net = dev_net(skb_dst(skb)->dev);
struct xfrm_state *x = skb_dst(skb)->xfrm;
int err; int err;
secpath_reset(skb);
if (xfrm_dev_offload_ok(skb, x)) {
struct sec_path *sp;
sp = secpath_dup(skb->sp);
if (!sp) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
kfree_skb(skb);
return -ENOMEM;
}
if (skb->sp)
secpath_put(skb->sp);
skb->sp = sp;
sp->olen++;
sp->xvec[skb->sp->len++] = x;
xfrm_state_hold(x);
if (skb_is_gso(skb)) {
skb_shinfo(skb)->gso_type |= SKB_GSO_ESP;
return xfrm_output2(net, sk, skb);
}
if (x->xso.dev && x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)
goto out;
}
if (skb_is_gso(skb)) if (skb_is_gso(skb))
return xfrm_output_gso(net, sk, skb); return xfrm_output_gso(net, sk, skb);
...@@ -214,6 +245,7 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb) ...@@ -214,6 +245,7 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb)
} }
} }
out:
return xfrm_output2(net, sk, skb); return xfrm_output2(net, sk, skb);
} }
EXPORT_SYMBOL_GPL(xfrm_output); EXPORT_SYMBOL_GPL(xfrm_output);
......
...@@ -116,8 +116,7 @@ static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short fa ...@@ -116,8 +116,7 @@ static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short fa
return afinfo; return afinfo;
} }
static inline struct dst_entry *__xfrm_dst_lookup(struct net *net, struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif,
int tos, int oif,
const xfrm_address_t *saddr, const xfrm_address_t *saddr,
const xfrm_address_t *daddr, const xfrm_address_t *daddr,
int family) int family)
...@@ -135,6 +134,7 @@ static inline struct dst_entry *__xfrm_dst_lookup(struct net *net, ...@@ -135,6 +134,7 @@ static inline struct dst_entry *__xfrm_dst_lookup(struct net *net,
return dst; return dst;
} }
EXPORT_SYMBOL(__xfrm_dst_lookup);
static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x, static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
int tos, int oif, int tos, int oif,
......
...@@ -440,6 +440,7 @@ static void xfrm_state_gc_destroy(struct xfrm_state *x) ...@@ -440,6 +440,7 @@ static void xfrm_state_gc_destroy(struct xfrm_state *x)
x->type->destructor(x); x->type->destructor(x);
xfrm_put_type(x->type); xfrm_put_type(x->type);
} }
xfrm_dev_state_free(x);
security_xfrm_state_free(x); security_xfrm_state_free(x);
kfree(x); kfree(x);
} }
...@@ -609,6 +610,8 @@ int __xfrm_state_delete(struct xfrm_state *x) ...@@ -609,6 +610,8 @@ int __xfrm_state_delete(struct xfrm_state *x)
net->xfrm.state_num--; net->xfrm.state_num--;
spin_unlock(&net->xfrm.xfrm_state_lock); spin_unlock(&net->xfrm.xfrm_state_lock);
xfrm_dev_state_delete(x);
/* All xfrm_state objects are created by xfrm_state_alloc. /* All xfrm_state objects are created by xfrm_state_alloc.
* The xfrm_state_alloc call gives a reference, and that * The xfrm_state_alloc call gives a reference, and that
* is what we are dropping here. * is what we are dropping here.
...@@ -653,12 +656,41 @@ xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) ...@@ -653,12 +656,41 @@ xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
return err; return err;
} }
static inline int
xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
{
int i, err = 0;
for (i = 0; i <= net->xfrm.state_hmask; i++) {
struct xfrm_state *x;
struct xfrm_state_offload *xso;
hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
xso = &x->xso;
if (xso->dev == dev &&
(err = security_xfrm_state_delete(x)) != 0) {
xfrm_audit_state_delete(x, 0, task_valid);
return err;
}
}
}
return err;
}
#else #else
static inline int static inline int
xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
{ {
return 0; return 0;
} }
static inline int
xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
{
return 0;
}
#endif #endif
int xfrm_state_flush(struct net *net, u8 proto, bool task_valid) int xfrm_state_flush(struct net *net, u8 proto, bool task_valid)
...@@ -701,6 +733,48 @@ int xfrm_state_flush(struct net *net, u8 proto, bool task_valid) ...@@ -701,6 +733,48 @@ int xfrm_state_flush(struct net *net, u8 proto, bool task_valid)
} }
EXPORT_SYMBOL(xfrm_state_flush); EXPORT_SYMBOL(xfrm_state_flush);
int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid)
{
int i, err = 0, cnt = 0;
spin_lock_bh(&net->xfrm.xfrm_state_lock);
err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid);
if (err)
goto out;
err = -ESRCH;
for (i = 0; i <= net->xfrm.state_hmask; i++) {
struct xfrm_state *x;
struct xfrm_state_offload *xso;
restart:
hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
xso = &x->xso;
if (!xfrm_state_kern(x) && xso->dev == dev) {
xfrm_state_hold(x);
spin_unlock_bh(&net->xfrm.xfrm_state_lock);
err = xfrm_state_delete(x);
xfrm_audit_state_delete(x, err ? 0 : 1,
task_valid);
xfrm_state_put(x);
if (!err)
cnt++;
spin_lock_bh(&net->xfrm.xfrm_state_lock);
goto restart;
}
}
}
if (cnt)
err = 0;
out:
spin_unlock_bh(&net->xfrm.xfrm_state_lock);
return err;
}
EXPORT_SYMBOL(xfrm_dev_state_flush);
void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si) void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si)
{ {
spin_lock_bh(&net->xfrm.xfrm_state_lock); spin_lock_bh(&net->xfrm.xfrm_state_lock);
......
...@@ -595,6 +595,10 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, ...@@ -595,6 +595,10 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
goto error; goto error;
} }
if (attrs[XFRMA_OFFLOAD_DEV] &&
xfrm_dev_state_add(net, x, nla_data(attrs[XFRMA_OFFLOAD_DEV])))
goto error;
if ((err = xfrm_alloc_replay_state_esn(&x->replay_esn, &x->preplay_esn, if ((err = xfrm_alloc_replay_state_esn(&x->replay_esn, &x->preplay_esn,
attrs[XFRMA_REPLAY_ESN_VAL]))) attrs[XFRMA_REPLAY_ESN_VAL])))
goto error; goto error;
...@@ -779,6 +783,23 @@ static int copy_sec_ctx(struct xfrm_sec_ctx *s, struct sk_buff *skb) ...@@ -779,6 +783,23 @@ static int copy_sec_ctx(struct xfrm_sec_ctx *s, struct sk_buff *skb)
return 0; return 0;
} }
static int copy_user_offload(struct xfrm_state_offload *xso, struct sk_buff *skb)
{
struct xfrm_user_offload *xuo;
struct nlattr *attr;
attr = nla_reserve(skb, XFRMA_OFFLOAD_DEV, sizeof(*xuo));
if (attr == NULL)
return -EMSGSIZE;
xuo = nla_data(attr);
xuo->ifindex = xso->dev->ifindex;
xuo->flags = xso->flags;
return 0;
}
static int copy_to_user_auth(struct xfrm_algo_auth *auth, struct sk_buff *skb) static int copy_to_user_auth(struct xfrm_algo_auth *auth, struct sk_buff *skb)
{ {
struct xfrm_algo *algo; struct xfrm_algo *algo;
...@@ -869,6 +890,10 @@ static int copy_to_user_state_extra(struct xfrm_state *x, ...@@ -869,6 +890,10 @@ static int copy_to_user_state_extra(struct xfrm_state *x,
&x->replay); &x->replay);
if (ret) if (ret)
goto out; goto out;
if(x->xso.dev)
ret = copy_user_offload(&x->xso, skb);
if (ret)
goto out;
if (x->security) if (x->security)
ret = copy_sec_ctx(x->security, skb); ret = copy_sec_ctx(x->security, skb);
out: out:
...@@ -2406,6 +2431,7 @@ static const struct nla_policy xfrma_policy[XFRMA_MAX+1] = { ...@@ -2406,6 +2431,7 @@ static const struct nla_policy xfrma_policy[XFRMA_MAX+1] = {
[XFRMA_SA_EXTRA_FLAGS] = { .type = NLA_U32 }, [XFRMA_SA_EXTRA_FLAGS] = { .type = NLA_U32 },
[XFRMA_PROTO] = { .type = NLA_U8 }, [XFRMA_PROTO] = { .type = NLA_U8 },
[XFRMA_ADDRESS_FILTER] = { .len = sizeof(struct xfrm_address_filter) }, [XFRMA_ADDRESS_FILTER] = { .len = sizeof(struct xfrm_address_filter) },
[XFRMA_OFFLOAD_DEV] = { .len = sizeof(struct xfrm_user_offload) },
}; };
static const struct nla_policy xfrma_spd_policy[XFRMA_SPD_MAX+1] = { static const struct nla_policy xfrma_spd_policy[XFRMA_SPD_MAX+1] = {
...@@ -2622,6 +2648,8 @@ static inline size_t xfrm_sa_len(struct xfrm_state *x) ...@@ -2622,6 +2648,8 @@ static inline size_t xfrm_sa_len(struct xfrm_state *x)
l += nla_total_size(sizeof(*x->coaddr)); l += nla_total_size(sizeof(*x->coaddr));
if (x->props.extra_flags) if (x->props.extra_flags)
l += nla_total_size(sizeof(x->props.extra_flags)); l += nla_total_size(sizeof(x->props.extra_flags));
if (x->xso.dev)
l += nla_total_size(sizeof(x->xso));
/* Must count x->lastused as it may become non-zero behind our back. */ /* Must count x->lastused as it may become non-zero behind our back. */
l += nla_total_size_64bit(sizeof(u64)); l += nla_total_size_64bit(sizeof(u64));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment