Commit d2d68ba9 authored by David S. Miller's avatar David S. Miller

ipv4: Cache input routes in fib_info nexthops.

Caching input routes is slightly simpler than output routes, since we
don't need to be concerned with nexthop exceptions.  (locally
destined, and routed packets, never trigger PMTU events or redirects
that will be processed by us).

However, we have to elide caching for the DIRECTSRC and non-zero itag
cases.
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent f2bb4bed
...@@ -82,6 +82,7 @@ struct fib_nh { ...@@ -82,6 +82,7 @@ struct fib_nh {
__be32 nh_saddr; __be32 nh_saddr;
int nh_saddr_genid; int nh_saddr_genid;
struct rtable *nh_rth_output; struct rtable *nh_rth_output;
struct rtable *nh_rth_input;
struct fnhe_hash_bucket *nh_exceptions; struct fnhe_hash_bucket *nh_exceptions;
}; };
......
...@@ -173,6 +173,8 @@ static void free_fib_info_rcu(struct rcu_head *head) ...@@ -173,6 +173,8 @@ static void free_fib_info_rcu(struct rcu_head *head)
free_nh_exceptions(nexthop_nh); free_nh_exceptions(nexthop_nh);
if (nexthop_nh->nh_rth_output) if (nexthop_nh->nh_rth_output)
dst_release(&nexthop_nh->nh_rth_output->dst); dst_release(&nexthop_nh->nh_rth_output->dst);
if (nexthop_nh->nh_rth_input)
dst_release(&nexthop_nh->nh_rth_input->dst);
} endfor_nexthops(fi); } endfor_nexthops(fi);
release_net(fi->fib_net); release_net(fi->fib_net);
......
...@@ -1231,6 +1231,9 @@ static void rt_cache_route(struct fib_nh *nh, struct rtable *rt) ...@@ -1231,6 +1231,9 @@ static void rt_cache_route(struct fib_nh *nh, struct rtable *rt)
{ {
struct rtable *orig, *prev, **p = &nh->nh_rth_output; struct rtable *orig, *prev, **p = &nh->nh_rth_output;
if (rt_is_input_route(rt))
p = &nh->nh_rth_input;
orig = *p; orig = *p;
prev = cmpxchg(p, orig, rt); prev = cmpxchg(p, orig, rt);
...@@ -1241,6 +1244,11 @@ static void rt_cache_route(struct fib_nh *nh, struct rtable *rt) ...@@ -1241,6 +1244,11 @@ static void rt_cache_route(struct fib_nh *nh, struct rtable *rt)
} }
} }
static bool rt_cache_valid(struct rtable *rt)
{
return (rt && rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK);
}
static void rt_set_nexthop(struct rtable *rt, __be32 daddr, static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
const struct fib_result *res, const struct fib_result *res,
struct fib_nh_exception *fnhe, struct fib_nh_exception *fnhe,
...@@ -1257,8 +1265,7 @@ static void rt_set_nexthop(struct rtable *rt, __be32 daddr, ...@@ -1257,8 +1265,7 @@ static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
#ifdef CONFIG_IP_ROUTE_CLASSID #ifdef CONFIG_IP_ROUTE_CLASSID
rt->dst.tclassid = nh->nh_tclassid; rt->dst.tclassid = nh->nh_tclassid;
#endif #endif
if (!(rt->dst.flags & DST_HOST) && if (!(rt->dst.flags & DST_HOST))
rt_is_output_route(rt))
rt_cache_route(nh, rt); rt_cache_route(nh, rt);
} }
...@@ -1384,11 +1391,11 @@ static int __mkroute_input(struct sk_buff *skb, ...@@ -1384,11 +1391,11 @@ static int __mkroute_input(struct sk_buff *skb,
__be32 daddr, __be32 saddr, u32 tos, __be32 daddr, __be32 saddr, u32 tos,
struct rtable **result) struct rtable **result)
{ {
struct fib_nh_exception *fnhe;
struct rtable *rth; struct rtable *rth;
int err; int err;
struct in_device *out_dev; struct in_device *out_dev;
unsigned int flags = 0; unsigned int flags = 0;
bool do_cache;
u32 itag; u32 itag;
/* get a working reference to the output device */ /* get a working reference to the output device */
...@@ -1431,13 +1438,21 @@ static int __mkroute_input(struct sk_buff *skb, ...@@ -1431,13 +1438,21 @@ static int __mkroute_input(struct sk_buff *skb,
} }
} }
fnhe = NULL; do_cache = false;
if (res->fi) if (res->fi) {
fnhe = find_exception(&FIB_RES_NH(*res), daddr); if (!(flags & RTCF_DIRECTSRC) && !itag) {
rth = FIB_RES_NH(*res).nh_rth_input;
if (rt_cache_valid(rth)) {
dst_use(&rth->dst, jiffies);
goto out;
}
do_cache = true;
}
}
rth = rt_dst_alloc(out_dev->dev, rth = rt_dst_alloc(out_dev->dev,
IN_DEV_CONF_GET(in_dev, NOPOLICY), IN_DEV_CONF_GET(in_dev, NOPOLICY),
IN_DEV_CONF_GET(out_dev, NOXFRM), false); IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
if (!rth) { if (!rth) {
err = -ENOBUFS; err = -ENOBUFS;
goto cleanup; goto cleanup;
...@@ -1456,8 +1471,8 @@ static int __mkroute_input(struct sk_buff *skb, ...@@ -1456,8 +1471,8 @@ static int __mkroute_input(struct sk_buff *skb,
rth->dst.input = ip_forward; rth->dst.input = ip_forward;
rth->dst.output = ip_output; rth->dst.output = ip_output;
rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag); rt_set_nexthop(rth, daddr, res, NULL, res->fi, res->type, itag);
out:
*result = rth; *result = rth;
err = 0; err = 0;
cleanup: cleanup:
...@@ -1509,6 +1524,7 @@ static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr, ...@@ -1509,6 +1524,7 @@ static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
struct rtable *rth; struct rtable *rth;
int err = -EINVAL; int err = -EINVAL;
struct net *net = dev_net(dev); struct net *net = dev_net(dev);
bool do_cache;
/* IP on this device is disabled. */ /* IP on this device is disabled. */
...@@ -1522,6 +1538,7 @@ static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr, ...@@ -1522,6 +1538,7 @@ static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr)) if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
goto martian_source; goto martian_source;
res.fi = NULL;
if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0)) if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
goto brd_input; goto brd_input;
...@@ -1597,8 +1614,20 @@ out: return err; ...@@ -1597,8 +1614,20 @@ out: return err;
RT_CACHE_STAT_INC(in_brd); RT_CACHE_STAT_INC(in_brd);
local_input: local_input:
do_cache = false;
if (res.fi) {
if (!(flags & RTCF_DIRECTSRC) && !itag) {
rth = FIB_RES_NH(res).nh_rth_input;
if (rt_cache_valid(rth)) {
dst_use(&rth->dst, jiffies);
goto set_and_out;
}
do_cache = true;
}
}
rth = rt_dst_alloc(net->loopback_dev, rth = rt_dst_alloc(net->loopback_dev,
IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false); IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
if (!rth) if (!rth)
goto e_nobufs; goto e_nobufs;
...@@ -1622,6 +1651,9 @@ out: return err; ...@@ -1622,6 +1651,9 @@ out: return err;
rth->dst.error= -err; rth->dst.error= -err;
rth->rt_flags &= ~RTCF_LOCAL; rth->rt_flags &= ~RTCF_LOCAL;
} }
if (do_cache)
rt_cache_route(&FIB_RES_NH(res), rth);
set_and_out:
skb_dst_set(skb, &rth->dst); skb_dst_set(skb, &rth->dst);
err = 0; err = 0;
goto out; goto out;
...@@ -1756,8 +1788,7 @@ static struct rtable *__mkroute_output(const struct fib_result *res, ...@@ -1756,8 +1788,7 @@ static struct rtable *__mkroute_output(const struct fib_result *res,
fnhe = find_exception(&FIB_RES_NH(*res), fl4->daddr); fnhe = find_exception(&FIB_RES_NH(*res), fl4->daddr);
if (!fnhe) { if (!fnhe) {
rth = FIB_RES_NH(*res).nh_rth_output; rth = FIB_RES_NH(*res).nh_rth_output;
if (rth && if (rt_cache_valid(rth)) {
rth->dst.obsolete == DST_OBSOLETE_FORCE_CHK) {
dst_use(&rth->dst, jiffies); dst_use(&rth->dst, jiffies);
return rth; return rth;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment