Commit 59cc1f61 authored by Jiri Kosina's avatar Jiri Kosina Committed by David S. Miller

net: sched: convert qdisc linked list to hashtable

Convert the per-device linked list into a hashtable. The primary
motivation for this change is that currently, we're not tracking all the
qdiscs in hierarchy (e.g. excluding default qdiscs), as the lookup
performed over the linked list by qdisc_match_from_root() is rather
expensive.

The ultimate goal is to get rid of hidden qdiscs completely, which will
bring much more determinism in user experience.
Reviewed-by: default avatarCong Wang <xiyou.wangcong@gmail.com>
Signed-off-by: default avatarJiri Kosina <jkosina@suse.cz>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e87a8f24
...@@ -52,6 +52,7 @@ ...@@ -52,6 +52,7 @@
#include <uapi/linux/netdevice.h> #include <uapi/linux/netdevice.h>
#include <uapi/linux/if_bonding.h> #include <uapi/linux/if_bonding.h>
#include <uapi/linux/pkt_cls.h> #include <uapi/linux/pkt_cls.h>
#include <linux/hashtable.h>
struct netpoll_info; struct netpoll_info;
struct device; struct device;
...@@ -1800,6 +1801,9 @@ struct net_device { ...@@ -1800,6 +1801,9 @@ struct net_device {
unsigned int num_tx_queues; unsigned int num_tx_queues;
unsigned int real_num_tx_queues; unsigned int real_num_tx_queues;
struct Qdisc *qdisc; struct Qdisc *qdisc;
#ifdef CONFIG_NET_SCHED
DECLARE_HASHTABLE (qdisc_hash, 4);
#endif
unsigned long tx_queue_len; unsigned long tx_queue_len;
spinlock_t tx_global_lock; spinlock_t tx_global_lock;
int watchdog_timeo; int watchdog_timeo;
......
...@@ -90,8 +90,8 @@ int unregister_qdisc(struct Qdisc_ops *qops); ...@@ -90,8 +90,8 @@ int unregister_qdisc(struct Qdisc_ops *qops);
void qdisc_get_default(char *id, size_t len); void qdisc_get_default(char *id, size_t len);
int qdisc_set_default(const char *id); int qdisc_set_default(const char *id);
void qdisc_list_add(struct Qdisc *q); void qdisc_hash_add(struct Qdisc *q);
void qdisc_list_del(struct Qdisc *q); void qdisc_hash_del(struct Qdisc *q);
struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle); struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle);
struct Qdisc *qdisc_lookup_class(struct net_device *dev, u32 handle); struct Qdisc *qdisc_lookup_class(struct net_device *dev, u32 handle);
struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,
......
...@@ -61,7 +61,7 @@ struct Qdisc { ...@@ -61,7 +61,7 @@ struct Qdisc {
u32 limit; u32 limit;
const struct Qdisc_ops *ops; const struct Qdisc_ops *ops;
struct qdisc_size_table __rcu *stab; struct qdisc_size_table __rcu *stab;
struct list_head list; struct hlist_node hash;
u32 handle; u32 handle;
u32 parent; u32 parent;
void *u32_node; void *u32_node;
......
...@@ -7629,6 +7629,9 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, ...@@ -7629,6 +7629,9 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
INIT_LIST_HEAD(&dev->all_adj_list.lower); INIT_LIST_HEAD(&dev->all_adj_list.lower);
INIT_LIST_HEAD(&dev->ptype_all); INIT_LIST_HEAD(&dev->ptype_all);
INIT_LIST_HEAD(&dev->ptype_specific); INIT_LIST_HEAD(&dev->ptype_specific);
#ifdef CONFIG_NET_SCHED
hash_init(dev->qdisc_hash);
#endif
dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
setup(dev); setup(dev);
......
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
#include <linux/hrtimer.h> #include <linux/hrtimer.h>
#include <linux/lockdep.h> #include <linux/lockdep.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/hashtable.h>
#include <net/net_namespace.h> #include <net/net_namespace.h>
#include <net/sock.h> #include <net/sock.h>
...@@ -263,33 +264,33 @@ static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle) ...@@ -263,33 +264,33 @@ static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
root->handle == handle) root->handle == handle)
return root; return root;
list_for_each_entry_rcu(q, &root->list, list) { hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle) {
if (q->handle == handle) if (q->handle == handle)
return q; return q;
} }
return NULL; return NULL;
} }
void qdisc_list_add(struct Qdisc *q) void qdisc_hash_add(struct Qdisc *q)
{ {
if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) { if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
struct Qdisc *root = qdisc_dev(q)->qdisc; struct Qdisc *root = qdisc_dev(q)->qdisc;
WARN_ON_ONCE(root == &noop_qdisc); WARN_ON_ONCE(root == &noop_qdisc);
ASSERT_RTNL(); ASSERT_RTNL();
list_add_tail_rcu(&q->list, &root->list); hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);
} }
} }
EXPORT_SYMBOL(qdisc_list_add); EXPORT_SYMBOL(qdisc_hash_add);
void qdisc_list_del(struct Qdisc *q) void qdisc_hash_del(struct Qdisc *q)
{ {
if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) { if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
ASSERT_RTNL(); ASSERT_RTNL();
list_del_rcu(&q->list); hash_del_rcu(&q->hash);
} }
} }
EXPORT_SYMBOL(qdisc_list_del); EXPORT_SYMBOL(qdisc_hash_del);
struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
{ {
...@@ -998,7 +999,7 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, ...@@ -998,7 +999,7 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
goto err_out4; goto err_out4;
} }
qdisc_list_add(sch); qdisc_hash_add(sch);
return sch; return sch;
} }
...@@ -1435,6 +1436,7 @@ static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb, ...@@ -1435,6 +1436,7 @@ static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
{ {
int ret = 0, q_idx = *q_idx_p; int ret = 0, q_idx = *q_idx_p;
struct Qdisc *q; struct Qdisc *q;
int b;
if (!root) if (!root)
return 0; return 0;
...@@ -1449,7 +1451,7 @@ static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb, ...@@ -1449,7 +1451,7 @@ static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
goto done; goto done;
q_idx++; q_idx++;
} }
list_for_each_entry(q, &root->list, list) { hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
if (q_idx < s_q_idx) { if (q_idx < s_q_idx) {
q_idx++; q_idx++;
continue; continue;
...@@ -1765,6 +1767,7 @@ static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb, ...@@ -1765,6 +1767,7 @@ static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
int *t_p, int s_t) int *t_p, int s_t)
{ {
struct Qdisc *q; struct Qdisc *q;
int b;
if (!root) if (!root)
return 0; return 0;
...@@ -1772,7 +1775,7 @@ static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb, ...@@ -1772,7 +1775,7 @@ static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0) if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
return -1; return -1;
list_for_each_entry(q, &root->list, list) { hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
return -1; return -1;
} }
......
...@@ -423,7 +423,6 @@ struct Qdisc noop_qdisc = { ...@@ -423,7 +423,6 @@ struct Qdisc noop_qdisc = {
.dequeue = noop_dequeue, .dequeue = noop_dequeue,
.flags = TCQ_F_BUILTIN, .flags = TCQ_F_BUILTIN,
.ops = &noop_qdisc_ops, .ops = &noop_qdisc_ops,
.list = LIST_HEAD_INIT(noop_qdisc.list),
.q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock), .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
.dev_queue = &noop_netdev_queue, .dev_queue = &noop_netdev_queue,
.running = SEQCNT_ZERO(noop_qdisc.running), .running = SEQCNT_ZERO(noop_qdisc.running),
...@@ -613,7 +612,6 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, ...@@ -613,7 +612,6 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p); sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
sch->padded = (char *) sch - (char *) p; sch->padded = (char *) sch - (char *) p;
} }
INIT_LIST_HEAD(&sch->list);
skb_queue_head_init(&sch->q); skb_queue_head_init(&sch->q);
spin_lock_init(&sch->busylock); spin_lock_init(&sch->busylock);
...@@ -700,7 +698,7 @@ void qdisc_destroy(struct Qdisc *qdisc) ...@@ -700,7 +698,7 @@ void qdisc_destroy(struct Qdisc *qdisc)
return; return;
#ifdef CONFIG_NET_SCHED #ifdef CONFIG_NET_SCHED
qdisc_list_del(qdisc); qdisc_hash_del(qdisc);
qdisc_put_stab(rtnl_dereference(qdisc->stab)); qdisc_put_stab(rtnl_dereference(qdisc->stab));
#endif #endif
...@@ -788,6 +786,10 @@ static void attach_default_qdiscs(struct net_device *dev) ...@@ -788,6 +786,10 @@ static void attach_default_qdiscs(struct net_device *dev)
qdisc->ops->attach(qdisc); qdisc->ops->attach(qdisc);
} }
} }
#ifdef CONFIG_NET_SCHED
if (dev->qdisc)
qdisc_hash_add(dev->qdisc);
#endif
} }
static void transition_one_qdisc(struct net_device *dev, static void transition_one_qdisc(struct net_device *dev,
......
...@@ -88,7 +88,7 @@ static void mq_attach(struct Qdisc *sch) ...@@ -88,7 +88,7 @@ static void mq_attach(struct Qdisc *sch)
qdisc_destroy(old); qdisc_destroy(old);
#ifdef CONFIG_NET_SCHED #ifdef CONFIG_NET_SCHED
if (ntx < dev->real_num_tx_queues) if (ntx < dev->real_num_tx_queues)
qdisc_list_add(qdisc); qdisc_hash_add(qdisc);
#endif #endif
} }
......
...@@ -182,7 +182,7 @@ static void mqprio_attach(struct Qdisc *sch) ...@@ -182,7 +182,7 @@ static void mqprio_attach(struct Qdisc *sch)
if (old) if (old)
qdisc_destroy(old); qdisc_destroy(old);
if (ntx < dev->real_num_tx_queues) if (ntx < dev->real_num_tx_queues)
qdisc_list_add(qdisc); qdisc_hash_add(qdisc);
} }
kfree(priv->qdiscs); kfree(priv->qdiscs);
priv->qdiscs = NULL; priv->qdiscs = NULL;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment