Commit ed76f5ed authored by Vlad Buslov's avatar Vlad Buslov Committed by David S. Miller

net: sched: protect filter_chain list with filter_chain_lock mutex

Extend tcf_chain with new filter_chain_lock mutex. Always lock the chain
when accessing filter_chain list, instead of relying on rtnl lock.
Dereference filter_chain with tcf_chain_dereference() lockdep macro to
verify that all users of chain_list have the lock taken.

Rearrange tp insert/remove code in tc_new_tfilter/tc_del_tfilter to execute
all necessary code while holding chain lock in order to prevent
invalidation of chain_info structure by potential concurrent change. This
also serializes calls to tcf_chain0_head_change(), which allows head change
callbacks to rely on filter_chain_lock for synchronization instead of rtnl
mutex.
Signed-off-by: default avatarVlad Buslov <vladbu@mellanox.com>
Acked-by: default avatarJiri Pirko <jiri@mellanox.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent a5654820
...@@ -341,6 +341,8 @@ struct qdisc_skb_cb { ...@@ -341,6 +341,8 @@ struct qdisc_skb_cb {
typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);
struct tcf_chain { struct tcf_chain {
/* Protects filter_chain. */
struct mutex filter_chain_lock;
struct tcf_proto __rcu *filter_chain; struct tcf_proto __rcu *filter_chain;
struct list_head list; struct list_head list;
struct tcf_block *block; struct tcf_block *block;
...@@ -374,6 +376,21 @@ struct tcf_block { ...@@ -374,6 +376,21 @@ struct tcf_block {
struct rcu_head rcu; struct rcu_head rcu;
}; };
#ifdef CONFIG_PROVE_LOCKING
static inline bool lockdep_tcf_chain_is_locked(struct tcf_chain *chain)
{
return lockdep_is_held(&chain->filter_chain_lock);
}
#else
static inline bool lockdep_tcf_chain_is_locked(struct tcf_block *chain)
{
return true;
}
#endif /* #ifdef CONFIG_PROVE_LOCKING */
#define tcf_chain_dereference(p, chain) \
rcu_dereference_protected(p, lockdep_tcf_chain_is_locked(chain))
static inline void tcf_block_offload_inc(struct tcf_block *block, u32 *flags) static inline void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
{ {
if (*flags & TCA_CLS_FLAGS_IN_HW) if (*flags & TCA_CLS_FLAGS_IN_HW)
......
...@@ -221,6 +221,7 @@ static struct tcf_chain *tcf_chain_create(struct tcf_block *block, ...@@ -221,6 +221,7 @@ static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
if (!chain) if (!chain)
return NULL; return NULL;
list_add_tail(&chain->list, &block->chain_list); list_add_tail(&chain->list, &block->chain_list);
mutex_init(&chain->filter_chain_lock);
chain->block = block; chain->block = block;
chain->index = chain_index; chain->index = chain_index;
chain->refcnt = 1; chain->refcnt = 1;
...@@ -280,6 +281,7 @@ static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block) ...@@ -280,6 +281,7 @@ static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
{ {
struct tcf_block *block = chain->block; struct tcf_block *block = chain->block;
mutex_destroy(&chain->filter_chain_lock);
kfree(chain); kfree(chain);
if (free_block) if (free_block)
tcf_block_destroy(block); tcf_block_destroy(block);
...@@ -443,9 +445,13 @@ static void tcf_chain_put_explicitly_created(struct tcf_chain *chain) ...@@ -443,9 +445,13 @@ static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
static void tcf_chain_flush(struct tcf_chain *chain) static void tcf_chain_flush(struct tcf_chain *chain)
{ {
struct tcf_proto *tp = rtnl_dereference(chain->filter_chain); struct tcf_proto *tp;
mutex_lock(&chain->filter_chain_lock);
tp = tcf_chain_dereference(chain->filter_chain, chain);
tcf_chain0_head_change(chain, NULL); tcf_chain0_head_change(chain, NULL);
mutex_unlock(&chain->filter_chain_lock);
while (tp) { while (tp) {
RCU_INIT_POINTER(chain->filter_chain, tp->next); RCU_INIT_POINTER(chain->filter_chain, tp->next);
tcf_proto_destroy(tp, NULL); tcf_proto_destroy(tp, NULL);
...@@ -785,11 +791,29 @@ tcf_chain0_head_change_cb_add(struct tcf_block *block, ...@@ -785,11 +791,29 @@ tcf_chain0_head_change_cb_add(struct tcf_block *block,
mutex_lock(&block->lock); mutex_lock(&block->lock);
chain0 = block->chain0.chain; chain0 = block->chain0.chain;
if (chain0 && chain0->filter_chain) if (chain0)
tcf_chain_head_change_item(item, chain0->filter_chain); tcf_chain_hold(chain0);
else
list_add(&item->list, &block->chain0.filter_chain_list);
mutex_unlock(&block->lock);
if (chain0) {
struct tcf_proto *tp_head;
mutex_lock(&chain0->filter_chain_lock);
tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
if (tp_head)
tcf_chain_head_change_item(item, tp_head);
mutex_lock(&block->lock);
list_add(&item->list, &block->chain0.filter_chain_list); list_add(&item->list, &block->chain0.filter_chain_list);
mutex_unlock(&block->lock); mutex_unlock(&block->lock);
mutex_unlock(&chain0->filter_chain_lock);
tcf_chain_put(chain0);
}
return 0; return 0;
} }
...@@ -1464,9 +1488,10 @@ struct tcf_chain_info { ...@@ -1464,9 +1488,10 @@ struct tcf_chain_info {
struct tcf_proto __rcu *next; struct tcf_proto __rcu *next;
}; };
static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain_info *chain_info) static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
struct tcf_chain_info *chain_info)
{ {
return rtnl_dereference(*chain_info->pprev); return tcf_chain_dereference(*chain_info->pprev, chain);
} }
static void tcf_chain_tp_insert(struct tcf_chain *chain, static void tcf_chain_tp_insert(struct tcf_chain *chain,
...@@ -1475,7 +1500,7 @@ static void tcf_chain_tp_insert(struct tcf_chain *chain, ...@@ -1475,7 +1500,7 @@ static void tcf_chain_tp_insert(struct tcf_chain *chain,
{ {
if (*chain_info->pprev == chain->filter_chain) if (*chain_info->pprev == chain->filter_chain)
tcf_chain0_head_change(chain, tp); tcf_chain0_head_change(chain, tp);
RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain_info)); RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
rcu_assign_pointer(*chain_info->pprev, tp); rcu_assign_pointer(*chain_info->pprev, tp);
tcf_chain_hold(chain); tcf_chain_hold(chain);
} }
...@@ -1484,7 +1509,7 @@ static void tcf_chain_tp_remove(struct tcf_chain *chain, ...@@ -1484,7 +1509,7 @@ static void tcf_chain_tp_remove(struct tcf_chain *chain,
struct tcf_chain_info *chain_info, struct tcf_chain_info *chain_info,
struct tcf_proto *tp) struct tcf_proto *tp)
{ {
struct tcf_proto *next = rtnl_dereference(chain_info->next); struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
if (tp == chain->filter_chain) if (tp == chain->filter_chain)
tcf_chain0_head_change(chain, next); tcf_chain0_head_change(chain, next);
...@@ -1502,7 +1527,8 @@ static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain, ...@@ -1502,7 +1527,8 @@ static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
/* Check the chain for existence of proto-tcf with this priority */ /* Check the chain for existence of proto-tcf with this priority */
for (pprev = &chain->filter_chain; for (pprev = &chain->filter_chain;
(tp = rtnl_dereference(*pprev)); pprev = &tp->next) { (tp = tcf_chain_dereference(*pprev, chain));
pprev = &tp->next) {
if (tp->prio >= prio) { if (tp->prio >= prio) {
if (tp->prio == prio) { if (tp->prio == prio) {
if (prio_allocate || if (prio_allocate ||
...@@ -1710,12 +1736,13 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, ...@@ -1710,12 +1736,13 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
goto errout; goto errout;
} }
mutex_lock(&chain->filter_chain_lock);
tp = tcf_chain_tp_find(chain, &chain_info, protocol, tp = tcf_chain_tp_find(chain, &chain_info, protocol,
prio, prio_allocate); prio, prio_allocate);
if (IS_ERR(tp)) { if (IS_ERR(tp)) {
NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
err = PTR_ERR(tp); err = PTR_ERR(tp);
goto errout; goto errout_locked;
} }
if (tp == NULL) { if (tp == NULL) {
...@@ -1724,29 +1751,37 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, ...@@ -1724,29 +1751,37 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
if (tca[TCA_KIND] == NULL || !protocol) { if (tca[TCA_KIND] == NULL || !protocol) {
NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified"); NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
err = -EINVAL; err = -EINVAL;
goto errout; goto errout_locked;
} }
if (!(n->nlmsg_flags & NLM_F_CREATE)) { if (!(n->nlmsg_flags & NLM_F_CREATE)) {
NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter"); NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
err = -ENOENT; err = -ENOENT;
goto errout; goto errout_locked;
} }
if (prio_allocate) if (prio_allocate)
prio = tcf_auto_prio(tcf_chain_tp_prev(&chain_info)); prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
&chain_info));
mutex_unlock(&chain->filter_chain_lock);
tp = tcf_proto_create(nla_data(tca[TCA_KIND]), tp = tcf_proto_create(nla_data(tca[TCA_KIND]),
protocol, prio, chain, extack); protocol, prio, chain, extack);
if (IS_ERR(tp)) { if (IS_ERR(tp)) {
err = PTR_ERR(tp); err = PTR_ERR(tp);
goto errout; goto errout;
} }
mutex_lock(&chain->filter_chain_lock);
tcf_chain_tp_insert(chain, &chain_info, tp);
mutex_unlock(&chain->filter_chain_lock);
tp_created = 1; tp_created = 1;
} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
err = -EINVAL; err = -EINVAL;
goto errout; goto errout_locked;
} else {
mutex_unlock(&chain->filter_chain_lock);
} }
fh = tp->ops->get(tp, t->tcm_handle); fh = tp->ops->get(tp, t->tcm_handle);
...@@ -1772,15 +1807,11 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, ...@@ -1772,15 +1807,11 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE, n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
extack); extack);
if (err == 0) { if (err == 0)
if (tp_created)
tcf_chain_tp_insert(chain, &chain_info, tp);
tfilter_notify(net, skb, n, tp, block, q, parent, fh, tfilter_notify(net, skb, n, tp, block, q, parent, fh,
RTM_NEWTFILTER, false); RTM_NEWTFILTER, false);
} else { else if (tp_created)
if (tp_created)
tcf_proto_destroy(tp, NULL); tcf_proto_destroy(tp, NULL);
}
errout: errout:
if (chain) if (chain)
...@@ -1790,6 +1821,10 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, ...@@ -1790,6 +1821,10 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
/* Replay the request. */ /* Replay the request. */
goto replay; goto replay;
return err; return err;
errout_locked:
mutex_unlock(&chain->filter_chain_lock);
goto errout;
} }
static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
...@@ -1865,31 +1900,34 @@ static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, ...@@ -1865,31 +1900,34 @@ static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
goto errout; goto errout;
} }
mutex_lock(&chain->filter_chain_lock);
tp = tcf_chain_tp_find(chain, &chain_info, protocol, tp = tcf_chain_tp_find(chain, &chain_info, protocol,
prio, false); prio, false);
if (!tp || IS_ERR(tp)) { if (!tp || IS_ERR(tp)) {
NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
err = tp ? PTR_ERR(tp) : -ENOENT; err = tp ? PTR_ERR(tp) : -ENOENT;
goto errout; goto errout_locked;
} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
err = -EINVAL; err = -EINVAL;
goto errout_locked;
} else if (t->tcm_handle == 0) {
tcf_chain_tp_remove(chain, &chain_info, tp);
mutex_unlock(&chain->filter_chain_lock);
tfilter_notify(net, skb, n, tp, block, q, parent, fh,
RTM_DELTFILTER, false);
tcf_proto_destroy(tp, extack);
err = 0;
goto errout; goto errout;
} }
mutex_unlock(&chain->filter_chain_lock);
fh = tp->ops->get(tp, t->tcm_handle); fh = tp->ops->get(tp, t->tcm_handle);
if (!fh) { if (!fh) {
if (t->tcm_handle == 0) {
tcf_chain_tp_remove(chain, &chain_info, tp);
tfilter_notify(net, skb, n, tp, block, q, parent, fh,
RTM_DELTFILTER, false);
tcf_proto_destroy(tp, extack);
err = 0;
} else {
NL_SET_ERR_MSG(extack, "Specified filter handle not found"); NL_SET_ERR_MSG(extack, "Specified filter handle not found");
err = -ENOENT; err = -ENOENT;
}
} else { } else {
bool last; bool last;
...@@ -1899,7 +1937,10 @@ static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, ...@@ -1899,7 +1937,10 @@ static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
if (err) if (err)
goto errout; goto errout;
if (last) { if (last) {
mutex_lock(&chain->filter_chain_lock);
tcf_chain_tp_remove(chain, &chain_info, tp); tcf_chain_tp_remove(chain, &chain_info, tp);
mutex_unlock(&chain->filter_chain_lock);
tcf_proto_destroy(tp, extack); tcf_proto_destroy(tp, extack);
} }
} }
...@@ -1909,6 +1950,10 @@ static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, ...@@ -1909,6 +1950,10 @@ static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
tcf_chain_put(chain); tcf_chain_put(chain);
tcf_block_release(q, block); tcf_block_release(q, block);
return err; return err;
errout_locked:
mutex_unlock(&chain->filter_chain_lock);
goto errout;
} }
static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n, static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
...@@ -1966,8 +2011,10 @@ static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n, ...@@ -1966,8 +2011,10 @@ static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
goto errout; goto errout;
} }
mutex_lock(&chain->filter_chain_lock);
tp = tcf_chain_tp_find(chain, &chain_info, protocol, tp = tcf_chain_tp_find(chain, &chain_info, protocol,
prio, false); prio, false);
mutex_unlock(&chain->filter_chain_lock);
if (!tp || IS_ERR(tp)) { if (!tp || IS_ERR(tp)) {
NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
err = tp ? PTR_ERR(tp) : -ENOENT; err = tp ? PTR_ERR(tp) : -ENOENT;
......
...@@ -1366,7 +1366,11 @@ static void mini_qdisc_rcu_func(struct rcu_head *head) ...@@ -1366,7 +1366,11 @@ static void mini_qdisc_rcu_func(struct rcu_head *head)
void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
struct tcf_proto *tp_head) struct tcf_proto *tp_head)
{ {
struct mini_Qdisc *miniq_old = rtnl_dereference(*miniqp->p_miniq); /* Protected with chain0->filter_chain_lock.
* Can't access chain directly because tp_head can be NULL.
*/
struct mini_Qdisc *miniq_old =
rcu_dereference_protected(*miniqp->p_miniq, 1);
struct mini_Qdisc *miniq; struct mini_Qdisc *miniq;
if (!tp_head) { if (!tp_head) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment