Commit f40b6ae2 authored by David Ahern's avatar David Ahern Committed by David S. Miller

ipv6: Move pcpu cached routes to fib6_nh

rt6_info are specific instances of a fib entry and are tied to a
device and gateway - ie., a nexthop. Before nexthop objects, IPv6 fib
entries have separate fib6_info for each nexthop in a multipath route,
so the location of the pcpu cache in the fib6_info struct worked.
However, with nexthop objects a fib6_info can point to a set of nexthops
(yet another alignment of ipv6 with ipv4). Accordingly, the pcpu
cache needs to be moved to the fib6_nh struct so the cached entries
are local to the nexthop specification used to create the rt6_info.

Initialization and free of the pcpu entries moved to fib6_nh_init and
fib6_nh_release.

Change in location only, from fib6_info down to fib6_nh; no other
functional change intended.
Signed-off-by: default avatarDavid Ahern <dsahern@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent daeceb2d
...@@ -131,6 +131,8 @@ struct fib6_nh { ...@@ -131,6 +131,8 @@ struct fib6_nh {
#ifdef CONFIG_IPV6_ROUTER_PREF #ifdef CONFIG_IPV6_ROUTER_PREF
unsigned long last_probe; unsigned long last_probe;
#endif #endif
struct rt6_info * __percpu *rt6i_pcpu;
}; };
struct fib6_info { struct fib6_info {
...@@ -156,7 +158,6 @@ struct fib6_info { ...@@ -156,7 +158,6 @@ struct fib6_info {
struct rt6key fib6_src; struct rt6key fib6_src;
struct rt6key fib6_prefsrc; struct rt6key fib6_prefsrc;
struct rt6_info * __percpu *rt6i_pcpu;
struct rt6_exception_bucket __rcu *rt6i_exception_bucket; struct rt6_exception_bucket __rcu *rt6i_exception_bucket;
u32 fib6_metric; u32 fib6_metric;
......
...@@ -6341,16 +6341,16 @@ void addrconf_disable_policy_idev(struct inet6_dev *idev, int val) ...@@ -6341,16 +6341,16 @@ void addrconf_disable_policy_idev(struct inet6_dev *idev, int val)
list_for_each_entry(ifa, &idev->addr_list, if_list) { list_for_each_entry(ifa, &idev->addr_list, if_list) {
spin_lock(&ifa->lock); spin_lock(&ifa->lock);
if (ifa->rt) { if (ifa->rt) {
struct fib6_info *rt = ifa->rt; struct fib6_nh *nh = &ifa->rt->fib6_nh;
int cpu; int cpu;
rcu_read_lock(); rcu_read_lock();
ifa->rt->dst_nopolicy = val ? true : false; ifa->rt->dst_nopolicy = val ? true : false;
if (rt->rt6i_pcpu) { if (nh->rt6i_pcpu) {
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
struct rt6_info **rtp; struct rt6_info **rtp;
rtp = per_cpu_ptr(rt->rt6i_pcpu, cpu); rtp = per_cpu_ptr(nh->rt6i_pcpu, cpu);
addrconf_set_nopolicy(*rtp, val); addrconf_set_nopolicy(*rtp, val);
} }
} }
......
...@@ -155,12 +155,6 @@ struct fib6_info *fib6_info_alloc(gfp_t gfp_flags) ...@@ -155,12 +155,6 @@ struct fib6_info *fib6_info_alloc(gfp_t gfp_flags)
if (!f6i) if (!f6i)
return NULL; return NULL;
f6i->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);
if (!f6i->rt6i_pcpu) {
kfree(f6i);
return NULL;
}
INIT_LIST_HEAD(&f6i->fib6_siblings); INIT_LIST_HEAD(&f6i->fib6_siblings);
refcount_set(&f6i->fib6_ref, 1); refcount_set(&f6i->fib6_ref, 1);
...@@ -177,25 +171,6 @@ void fib6_info_destroy_rcu(struct rcu_head *head) ...@@ -177,25 +171,6 @@ void fib6_info_destroy_rcu(struct rcu_head *head)
bucket = rcu_dereference_protected(f6i->rt6i_exception_bucket, 1); bucket = rcu_dereference_protected(f6i->rt6i_exception_bucket, 1);
kfree(bucket); kfree(bucket);
if (f6i->rt6i_pcpu) {
int cpu;
for_each_possible_cpu(cpu) {
struct rt6_info **ppcpu_rt;
struct rt6_info *pcpu_rt;
ppcpu_rt = per_cpu_ptr(f6i->rt6i_pcpu, cpu);
pcpu_rt = *ppcpu_rt;
if (pcpu_rt) {
dst_dev_put(&pcpu_rt->dst);
dst_release(&pcpu_rt->dst);
*ppcpu_rt = NULL;
}
}
free_percpu(f6i->rt6i_pcpu);
}
fib6_nh_release(&f6i->fib6_nh); fib6_nh_release(&f6i->fib6_nh);
ip_fib_metrics_put(f6i->fib6_metrics); ip_fib_metrics_put(f6i->fib6_metrics);
...@@ -902,8 +877,12 @@ static struct fib6_node *fib6_add_1(struct net *net, ...@@ -902,8 +877,12 @@ static struct fib6_node *fib6_add_1(struct net *net,
static void fib6_drop_pcpu_from(struct fib6_info *f6i, static void fib6_drop_pcpu_from(struct fib6_info *f6i,
const struct fib6_table *table) const struct fib6_table *table)
{ {
struct fib6_nh *fib6_nh = &f6i->fib6_nh;
int cpu; int cpu;
if (!fib6_nh->rt6i_pcpu)
return;
/* Make sure rt6_make_pcpu_route() wont add other percpu routes /* Make sure rt6_make_pcpu_route() wont add other percpu routes
* while we are cleaning them here. * while we are cleaning them here.
*/ */
...@@ -917,7 +896,7 @@ static void fib6_drop_pcpu_from(struct fib6_info *f6i, ...@@ -917,7 +896,7 @@ static void fib6_drop_pcpu_from(struct fib6_info *f6i,
struct rt6_info **ppcpu_rt; struct rt6_info **ppcpu_rt;
struct rt6_info *pcpu_rt; struct rt6_info *pcpu_rt;
ppcpu_rt = per_cpu_ptr(f6i->rt6i_pcpu, cpu); ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
pcpu_rt = *ppcpu_rt; pcpu_rt = *ppcpu_rt;
if (pcpu_rt) { if (pcpu_rt) {
struct fib6_info *from; struct fib6_info *from;
...@@ -933,7 +912,6 @@ static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn, ...@@ -933,7 +912,6 @@ static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn,
{ {
struct fib6_table *table = rt->fib6_table; struct fib6_table *table = rt->fib6_table;
if (rt->rt6i_pcpu)
fib6_drop_pcpu_from(rt, table); fib6_drop_pcpu_from(rt, table);
if (refcount_read(&rt->fib6_ref) != 1) { if (refcount_read(&rt->fib6_ref) != 1) {
......
...@@ -1270,7 +1270,7 @@ static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res) ...@@ -1270,7 +1270,7 @@ static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res)
{ {
struct rt6_info *pcpu_rt, **p; struct rt6_info *pcpu_rt, **p;
p = this_cpu_ptr(res->f6i->rt6i_pcpu); p = this_cpu_ptr(res->nh->rt6i_pcpu);
pcpu_rt = *p; pcpu_rt = *p;
if (pcpu_rt) if (pcpu_rt)
...@@ -1291,7 +1291,7 @@ static struct rt6_info *rt6_make_pcpu_route(struct net *net, ...@@ -1291,7 +1291,7 @@ static struct rt6_info *rt6_make_pcpu_route(struct net *net,
} }
dst_hold(&pcpu_rt->dst); dst_hold(&pcpu_rt->dst);
p = this_cpu_ptr(res->f6i->rt6i_pcpu); p = this_cpu_ptr(res->nh->rt6i_pcpu);
prev = cmpxchg(p, NULL, pcpu_rt); prev = cmpxchg(p, NULL, pcpu_rt);
BUG_ON(prev); BUG_ON(prev);
...@@ -3068,6 +3068,12 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, ...@@ -3068,6 +3068,12 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,
!netif_carrier_ok(dev)) !netif_carrier_ok(dev))
fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);
if (!fib6_nh->rt6i_pcpu) {
err = -ENOMEM;
goto out;
}
err = fib_nh_common_init(&fib6_nh->nh_common, cfg->fc_encap, err = fib_nh_common_init(&fib6_nh->nh_common, cfg->fc_encap,
cfg->fc_encap_type, cfg, gfp_flags, extack); cfg->fc_encap_type, cfg, gfp_flags, extack);
if (err) if (err)
...@@ -3092,6 +3098,25 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, ...@@ -3092,6 +3098,25 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,
void fib6_nh_release(struct fib6_nh *fib6_nh) void fib6_nh_release(struct fib6_nh *fib6_nh)
{ {
if (fib6_nh->rt6i_pcpu) {
int cpu;
for_each_possible_cpu(cpu) {
struct rt6_info **ppcpu_rt;
struct rt6_info *pcpu_rt;
ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
pcpu_rt = *ppcpu_rt;
if (pcpu_rt) {
dst_dev_put(&pcpu_rt->dst);
dst_release(&pcpu_rt->dst);
*ppcpu_rt = NULL;
}
}
free_percpu(fib6_nh->rt6i_pcpu);
}
fib_nh_common_release(&fib6_nh->nh_common); fib_nh_common_release(&fib6_nh->nh_common);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment