Commit 0a5c0475 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

fib: add __rcu annotations

Add __rcu annotations and lockdep checks.

Add const qualifiers

node_parent() and node_parent_rcu() can use
rcu_dereference_index_check()
Signed-off-by: default avatarEric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent ab392d2d
...@@ -126,7 +126,7 @@ struct tnode { ...@@ -126,7 +126,7 @@ struct tnode {
struct work_struct work; struct work_struct work;
struct tnode *tnode_free; struct tnode *tnode_free;
}; };
struct rt_trie_node *child[0]; struct rt_trie_node __rcu *child[0];
}; };
#ifdef CONFIG_IP_FIB_TRIE_STATS #ifdef CONFIG_IP_FIB_TRIE_STATS
...@@ -151,7 +151,7 @@ struct trie_stat { ...@@ -151,7 +151,7 @@ struct trie_stat {
}; };
struct trie { struct trie {
struct rt_trie_node *trie; struct rt_trie_node __rcu *trie;
#ifdef CONFIG_IP_FIB_TRIE_STATS #ifdef CONFIG_IP_FIB_TRIE_STATS
struct trie_use_stats stats; struct trie_use_stats stats;
#endif #endif
...@@ -177,16 +177,29 @@ static const int sync_pages = 128; ...@@ -177,16 +177,29 @@ static const int sync_pages = 128;
static struct kmem_cache *fn_alias_kmem __read_mostly; static struct kmem_cache *fn_alias_kmem __read_mostly;
static struct kmem_cache *trie_leaf_kmem __read_mostly; static struct kmem_cache *trie_leaf_kmem __read_mostly;
static inline struct tnode *node_parent(struct rt_trie_node *node) /*
* caller must hold RTNL
*/
static inline struct tnode *node_parent(const struct rt_trie_node *node)
{ {
return (struct tnode *)(node->parent & ~NODE_TYPE_MASK); unsigned long parent;
parent = rcu_dereference_index_check(node->parent, lockdep_rtnl_is_held());
return (struct tnode *)(parent & ~NODE_TYPE_MASK);
} }
static inline struct tnode *node_parent_rcu(struct rt_trie_node *node) /*
* caller must hold RCU read lock or RTNL
*/
static inline struct tnode *node_parent_rcu(const struct rt_trie_node *node)
{ {
struct tnode *ret = node_parent(node); unsigned long parent;
parent = rcu_dereference_index_check(node->parent, rcu_read_lock_held() ||
lockdep_rtnl_is_held());
return rcu_dereference_rtnl(ret); return (struct tnode *)(parent & ~NODE_TYPE_MASK);
} }
/* Same as rcu_assign_pointer /* Same as rcu_assign_pointer
...@@ -198,18 +211,24 @@ static inline void node_set_parent(struct rt_trie_node *node, struct tnode *ptr) ...@@ -198,18 +211,24 @@ static inline void node_set_parent(struct rt_trie_node *node, struct tnode *ptr)
node->parent = (unsigned long)ptr | NODE_TYPE(node); node->parent = (unsigned long)ptr | NODE_TYPE(node);
} }
static inline struct rt_trie_node *tnode_get_child(struct tnode *tn, unsigned int i) /*
* caller must hold RTNL
*/
static inline struct rt_trie_node *tnode_get_child(const struct tnode *tn, unsigned int i)
{ {
BUG_ON(i >= 1U << tn->bits); BUG_ON(i >= 1U << tn->bits);
return tn->child[i]; return rtnl_dereference(tn->child[i]);
} }
static inline struct rt_trie_node *tnode_get_child_rcu(struct tnode *tn, unsigned int i) /*
* caller must hold RCU read lock or RTNL
*/
static inline struct rt_trie_node *tnode_get_child_rcu(const struct tnode *tn, unsigned int i)
{ {
struct rt_trie_node *ret = tnode_get_child(tn, i); BUG_ON(i >= 1U << tn->bits);
return rcu_dereference_rtnl(ret); return rcu_dereference_rtnl(tn->child[i]);
} }
static inline int tnode_child_length(const struct tnode *tn) static inline int tnode_child_length(const struct tnode *tn)
...@@ -487,7 +506,7 @@ static inline void put_child(struct trie *t, struct tnode *tn, int i, ...@@ -487,7 +506,7 @@ static inline void put_child(struct trie *t, struct tnode *tn, int i,
static void tnode_put_child_reorg(struct tnode *tn, int i, struct rt_trie_node *n, static void tnode_put_child_reorg(struct tnode *tn, int i, struct rt_trie_node *n,
int wasfull) int wasfull)
{ {
struct rt_trie_node *chi = tn->child[i]; struct rt_trie_node *chi = rtnl_dereference(tn->child[i]);
int isfull; int isfull;
BUG_ON(i >= 1<<tn->bits); BUG_ON(i >= 1<<tn->bits);
...@@ -665,7 +684,7 @@ static struct rt_trie_node *resize(struct trie *t, struct tnode *tn) ...@@ -665,7 +684,7 @@ static struct rt_trie_node *resize(struct trie *t, struct tnode *tn)
for (i = 0; i < tnode_child_length(tn); i++) { for (i = 0; i < tnode_child_length(tn); i++) {
struct rt_trie_node *n; struct rt_trie_node *n;
n = tn->child[i]; n = rtnl_dereference(tn->child[i]);
if (!n) if (!n)
continue; continue;
...@@ -679,6 +698,20 @@ static struct rt_trie_node *resize(struct trie *t, struct tnode *tn) ...@@ -679,6 +698,20 @@ static struct rt_trie_node *resize(struct trie *t, struct tnode *tn)
return (struct rt_trie_node *) tn; return (struct rt_trie_node *) tn;
} }
static void tnode_clean_free(struct tnode *tn)
{
int i;
struct tnode *tofree;
for (i = 0; i < tnode_child_length(tn); i++) {
tofree = (struct tnode *)rtnl_dereference(tn->child[i]);
if (tofree)
tnode_free(tofree);
}
tnode_free(tn);
}
static struct tnode *inflate(struct trie *t, struct tnode *tn) static struct tnode *inflate(struct trie *t, struct tnode *tn)
{ {
struct tnode *oldtnode = tn; struct tnode *oldtnode = tn;
...@@ -755,8 +788,8 @@ static struct tnode *inflate(struct trie *t, struct tnode *tn) ...@@ -755,8 +788,8 @@ static struct tnode *inflate(struct trie *t, struct tnode *tn)
inode = (struct tnode *) node; inode = (struct tnode *) node;
if (inode->bits == 1) { if (inode->bits == 1) {
put_child(t, tn, 2*i, inode->child[0]); put_child(t, tn, 2*i, rtnl_dereference(inode->child[0]));
put_child(t, tn, 2*i+1, inode->child[1]); put_child(t, tn, 2*i+1, rtnl_dereference(inode->child[1]));
tnode_free_safe(inode); tnode_free_safe(inode);
continue; continue;
...@@ -797,8 +830,8 @@ static struct tnode *inflate(struct trie *t, struct tnode *tn) ...@@ -797,8 +830,8 @@ static struct tnode *inflate(struct trie *t, struct tnode *tn)
size = tnode_child_length(left); size = tnode_child_length(left);
for (j = 0; j < size; j++) { for (j = 0; j < size; j++) {
put_child(t, left, j, inode->child[j]); put_child(t, left, j, rtnl_dereference(inode->child[j]));
put_child(t, right, j, inode->child[j + size]); put_child(t, right, j, rtnl_dereference(inode->child[j + size]));
} }
put_child(t, tn, 2*i, resize(t, left)); put_child(t, tn, 2*i, resize(t, left));
put_child(t, tn, 2*i+1, resize(t, right)); put_child(t, tn, 2*i+1, resize(t, right));
...@@ -808,18 +841,8 @@ static struct tnode *inflate(struct trie *t, struct tnode *tn) ...@@ -808,18 +841,8 @@ static struct tnode *inflate(struct trie *t, struct tnode *tn)
tnode_free_safe(oldtnode); tnode_free_safe(oldtnode);
return tn; return tn;
nomem: nomem:
{ tnode_clean_free(tn);
int size = tnode_child_length(tn);
int j;
for (j = 0; j < size; j++)
if (tn->child[j])
tnode_free((struct tnode *)tn->child[j]);
tnode_free(tn);
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
}
} }
static struct tnode *halve(struct trie *t, struct tnode *tn) static struct tnode *halve(struct trie *t, struct tnode *tn)
...@@ -890,18 +913,8 @@ static struct tnode *halve(struct trie *t, struct tnode *tn) ...@@ -890,18 +913,8 @@ static struct tnode *halve(struct trie *t, struct tnode *tn)
tnode_free_safe(oldtnode); tnode_free_safe(oldtnode);
return tn; return tn;
nomem: nomem:
{ tnode_clean_free(tn);
int size = tnode_child_length(tn);
int j;
for (j = 0; j < size; j++)
if (tn->child[j])
tnode_free((struct tnode *)tn->child[j]);
tnode_free(tn);
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
}
} }
/* readside must use rcu_read_lock currently dump routines /* readside must use rcu_read_lock currently dump routines
...@@ -1033,7 +1046,7 @@ static struct list_head *fib_insert_node(struct trie *t, u32 key, int plen) ...@@ -1033,7 +1046,7 @@ static struct list_head *fib_insert_node(struct trie *t, u32 key, int plen)
t_key cindex; t_key cindex;
pos = 0; pos = 0;
n = t->trie; n = rtnl_dereference(t->trie);
/* If we point to NULL, stop. Either the tree is empty and we should /* If we point to NULL, stop. Either the tree is empty and we should
* just put a new leaf in if, or we have reached an empty child slot, * just put a new leaf in if, or we have reached an empty child slot,
...@@ -1756,7 +1769,7 @@ static struct leaf *leaf_walk_rcu(struct tnode *p, struct rt_trie_node *c) ...@@ -1756,7 +1769,7 @@ static struct leaf *leaf_walk_rcu(struct tnode *p, struct rt_trie_node *c)
continue; continue;
if (IS_LEAF(c)) { if (IS_LEAF(c)) {
prefetch(p->child[idx]); prefetch(rcu_dereference_rtnl(p->child[idx]));
return (struct leaf *) c; return (struct leaf *) c;
} }
...@@ -2272,7 +2285,7 @@ static void *fib_trie_seq_next(struct seq_file *seq, void *v, loff_t *pos) ...@@ -2272,7 +2285,7 @@ static void *fib_trie_seq_next(struct seq_file *seq, void *v, loff_t *pos)
/* walk rest of this hash chain */ /* walk rest of this hash chain */
h = tb->tb_id & (FIB_TABLE_HASHSZ - 1); h = tb->tb_id & (FIB_TABLE_HASHSZ - 1);
while ( (tb_node = rcu_dereference(tb->tb_hlist.next)) ) { while ((tb_node = rcu_dereference(hlist_next_rcu(&tb->tb_hlist)))) {
tb = hlist_entry(tb_node, struct fib_table, tb_hlist); tb = hlist_entry(tb_node, struct fib_table, tb_hlist);
n = fib_trie_get_first(iter, (struct trie *) tb->tb_data); n = fib_trie_get_first(iter, (struct trie *) tb->tb_data);
if (n) if (n)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment