Commit 8fb472c0 authored by Nikolay Aleksandrov's avatar Nikolay Aleksandrov Committed by David S. Miller

ipmr: improve hash scalability

Recently we started using ipmr with thousands of entries and easily hit
soft lockups on smaller devices. The reason is that the hash function
uses the high order bits from the src and dst, but those don't change in
many common cases, also the hash table  is only 64 elements so with
thousands it doesn't scale at all.
This patch migrates the hash table to rhashtable, and in particular the
rhl interface which allows for duplicate elements to be chained because
of the MFC_PROXY support (*,G; *,*,oif cases) which allows for multiple
duplicate entries to be added with different interfaces (IMO wrong, but
it's been in for a long time).

And here are some results from tests I've run in a VM:
 mr_table size (default, allocated for all namespaces):
  Before                    After
   49304 bytes               2400 bytes

 Add 65000 routes (the diff is much larger on smaller devices):
  Before                    After
   1m42s                     58s

 Forwarding 256 byte packets with 65000 routes (test done in a VM):
  Before                    After
   3 Mbps / ~1465 pps        122 Mbps / ~59000 pps

As a bonus we no longer see the soft lockups on smaller devices which
showed up even with 2000 entries before.
Signed-off-by: default avatarNikolay Aleksandrov <nikolay@cumulusnetworks.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent c1ce1560
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
#include <linux/in.h> #include <linux/in.h>
#include <linux/pim.h> #include <linux/pim.h>
#include <linux/rhashtable.h>
#include <net/sock.h> #include <net/sock.h>
#include <uapi/linux/mroute.h> #include <uapi/linux/mroute.h>
...@@ -60,7 +61,6 @@ struct vif_device { ...@@ -60,7 +61,6 @@ struct vif_device {
#define VIFF_STATIC 0x8000 #define VIFF_STATIC 0x8000
#define VIF_EXISTS(_mrt, _idx) ((_mrt)->vif_table[_idx].dev != NULL) #define VIF_EXISTS(_mrt, _idx) ((_mrt)->vif_table[_idx].dev != NULL)
#define MFC_LINES 64
struct mr_table { struct mr_table {
struct list_head list; struct list_head list;
...@@ -69,8 +69,9 @@ struct mr_table { ...@@ -69,8 +69,9 @@ struct mr_table {
struct sock __rcu *mroute_sk; struct sock __rcu *mroute_sk;
struct timer_list ipmr_expire_timer; struct timer_list ipmr_expire_timer;
struct list_head mfc_unres_queue; struct list_head mfc_unres_queue;
struct list_head mfc_cache_array[MFC_LINES];
struct vif_device vif_table[MAXVIFS]; struct vif_device vif_table[MAXVIFS];
struct rhltable mfc_hash;
struct list_head mfc_cache_list;
int maxvif; int maxvif;
atomic_t cache_resolve_queue_len; atomic_t cache_resolve_queue_len;
bool mroute_do_assert; bool mroute_do_assert;
...@@ -85,17 +86,48 @@ enum { ...@@ -85,17 +86,48 @@ enum {
MFC_STATIC = BIT(0), MFC_STATIC = BIT(0),
}; };
struct mfc_cache_cmp_arg {
__be32 mfc_mcastgrp;
__be32 mfc_origin;
};
/**
* struct mfc_cache - multicast routing entries
* @mnode: rhashtable list
* @mfc_mcastgrp: destination multicast group address
* @mfc_origin: source address
* @cmparg: used for rhashtable comparisons
* @mfc_parent: source interface (iif)
* @mfc_flags: entry flags
* @expires: unresolved entry expire time
* @unresolved: unresolved cached skbs
* @last_assert: time of last assert
* @minvif: minimum VIF id
* @maxvif: maximum VIF id
* @bytes: bytes that have passed for this entry
* @pkt: packets that have passed for this entry
* @wrong_if: number of wrong source interface hits
* @lastuse: time of last use of the group (traffic or update)
* @ttls: OIF TTL threshold array
* @list: global entry list
* @rcu: used for entry destruction
*/
struct mfc_cache { struct mfc_cache {
struct list_head list; struct rhlist_head mnode;
__be32 mfc_mcastgrp; /* Group the entry belongs to */ union {
__be32 mfc_origin; /* Source of packet */ struct {
vifi_t mfc_parent; /* Source interface */ __be32 mfc_mcastgrp;
int mfc_flags; /* Flags on line */ __be32 mfc_origin;
};
struct mfc_cache_cmp_arg cmparg;
};
vifi_t mfc_parent;
int mfc_flags;
union { union {
struct { struct {
unsigned long expires; unsigned long expires;
struct sk_buff_head unresolved; /* Unresolved buffers */ struct sk_buff_head unresolved;
} unres; } unres;
struct { struct {
unsigned long last_assert; unsigned long last_assert;
...@@ -105,18 +137,13 @@ struct mfc_cache { ...@@ -105,18 +137,13 @@ struct mfc_cache {
unsigned long pkt; unsigned long pkt;
unsigned long wrong_if; unsigned long wrong_if;
unsigned long lastuse; unsigned long lastuse;
unsigned char ttls[MAXVIFS]; /* TTL thresholds */ unsigned char ttls[MAXVIFS];
} res; } res;
} mfc_un; } mfc_un;
struct list_head list;
struct rcu_head rcu; struct rcu_head rcu;
}; };
#ifdef __BIG_ENDIAN
#define MFC_HASH(a,b) (((((__force u32)(__be32)a)>>24)^(((__force u32)(__be32)b)>>26))&(MFC_LINES-1))
#else
#define MFC_HASH(a,b) ((((__force u32)(__be32)a)^(((__force u32)(__be32)b)>>2))&(MFC_LINES-1))
#endif
struct rtmsg; struct rtmsg;
int ipmr_get_route(struct net *net, struct sk_buff *skb, int ipmr_get_route(struct net *net, struct sk_buff *skb,
__be32 saddr, __be32 daddr, __be32 saddr, __be32 daddr,
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment