Commit cdfafe98 authored by Björn Töpel's avatar Björn Töpel Committed by Alexei Starovoitov

xdp: Make cpumap flush_list common for all map instances

The cpumap flush list is used to track entries that need to flushed
from via the xdp_do_flush_map() function. This list used to be
per-map, but there is really no reason for that. Instead make the
flush list global for all devmaps, which simplifies __cpu_map_flush()
and cpu_map_alloc().
Signed-off-by: default avatarBjörn Töpel <bjorn.topel@intel.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarToke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/bpf/20191219061006.21980-7-bjorn.topel@gmail.com
parent 96360004
...@@ -966,7 +966,7 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, ...@@ -966,7 +966,7 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
struct bpf_prog *xdp_prog); struct bpf_prog *xdp_prog);
struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);
void __cpu_map_flush(struct bpf_map *map); void __cpu_map_flush(void);
int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
struct net_device *dev_rx); struct net_device *dev_rx);
...@@ -1097,7 +1097,7 @@ struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) ...@@ -1097,7 +1097,7 @@ struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
return NULL; return NULL;
} }
static inline void __cpu_map_flush(struct bpf_map *map) static inline void __cpu_map_flush(void)
{ {
} }
......
...@@ -72,17 +72,18 @@ struct bpf_cpu_map { ...@@ -72,17 +72,18 @@ struct bpf_cpu_map {
struct bpf_map map; struct bpf_map map;
/* Below members specific for map type */ /* Below members specific for map type */
struct bpf_cpu_map_entry **cpu_map; struct bpf_cpu_map_entry **cpu_map;
struct list_head __percpu *flush_list;
}; };
static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
static int bq_flush_to_queue(struct xdp_bulk_queue *bq); static int bq_flush_to_queue(struct xdp_bulk_queue *bq);
static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
{ {
struct bpf_cpu_map *cmap; struct bpf_cpu_map *cmap;
int err = -ENOMEM; int err = -ENOMEM;
int ret, cpu;
u64 cost; u64 cost;
int ret;
if (!capable(CAP_SYS_ADMIN)) if (!capable(CAP_SYS_ADMIN))
return ERR_PTR(-EPERM); return ERR_PTR(-EPERM);
...@@ -106,7 +107,6 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) ...@@ -106,7 +107,6 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
/* make sure page count doesn't overflow */ /* make sure page count doesn't overflow */
cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *); cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);
cost += sizeof(struct list_head) * num_possible_cpus();
/* Notice returns -EPERM on if map size is larger than memlock limit */ /* Notice returns -EPERM on if map size is larger than memlock limit */
ret = bpf_map_charge_init(&cmap->map.memory, cost); ret = bpf_map_charge_init(&cmap->map.memory, cost);
...@@ -115,23 +115,14 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) ...@@ -115,23 +115,14 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
goto free_cmap; goto free_cmap;
} }
cmap->flush_list = alloc_percpu(struct list_head);
if (!cmap->flush_list)
goto free_charge;
for_each_possible_cpu(cpu)
INIT_LIST_HEAD(per_cpu_ptr(cmap->flush_list, cpu));
/* Alloc array for possible remote "destination" CPUs */ /* Alloc array for possible remote "destination" CPUs */
cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries * cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
sizeof(struct bpf_cpu_map_entry *), sizeof(struct bpf_cpu_map_entry *),
cmap->map.numa_node); cmap->map.numa_node);
if (!cmap->cpu_map) if (!cmap->cpu_map)
goto free_percpu; goto free_charge;
return &cmap->map; return &cmap->map;
free_percpu:
free_percpu(cmap->flush_list);
free_charge: free_charge:
bpf_map_charge_finish(&cmap->map.memory); bpf_map_charge_finish(&cmap->map.memory);
free_cmap: free_cmap:
...@@ -526,7 +517,6 @@ static void cpu_map_free(struct bpf_map *map) ...@@ -526,7 +517,6 @@ static void cpu_map_free(struct bpf_map *map)
/* bq flush and cleanup happens after RCU grace-period */ /* bq flush and cleanup happens after RCU grace-period */
__cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */ __cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
} }
free_percpu(cmap->flush_list);
bpf_map_area_free(cmap->cpu_map); bpf_map_area_free(cmap->cpu_map);
kfree(cmap); kfree(cmap);
} }
...@@ -618,7 +608,7 @@ static int bq_flush_to_queue(struct xdp_bulk_queue *bq) ...@@ -618,7 +608,7 @@ static int bq_flush_to_queue(struct xdp_bulk_queue *bq)
*/ */
static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
{ {
struct list_head *flush_list = this_cpu_ptr(rcpu->cmap->flush_list); struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq); struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
if (unlikely(bq->count == CPU_MAP_BULK_SIZE)) if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
...@@ -657,10 +647,9 @@ int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, ...@@ -657,10 +647,9 @@ int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
return 0; return 0;
} }
void __cpu_map_flush(struct bpf_map *map) void __cpu_map_flush(void)
{ {
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map); struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
struct list_head *flush_list = this_cpu_ptr(cmap->flush_list);
struct xdp_bulk_queue *bq, *tmp; struct xdp_bulk_queue *bq, *tmp;
list_for_each_entry_safe(bq, tmp, flush_list, flush_node) { list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
...@@ -670,3 +659,14 @@ void __cpu_map_flush(struct bpf_map *map) ...@@ -670,3 +659,14 @@ void __cpu_map_flush(struct bpf_map *map)
wake_up_process(bq->obj->kthread); wake_up_process(bq->obj->kthread);
} }
} }
static int __init cpu_map_init(void)
{
int cpu;
for_each_possible_cpu(cpu)
INIT_LIST_HEAD(&per_cpu(cpu_map_flush_list, cpu));
return 0;
}
subsys_initcall(cpu_map_init);
...@@ -3558,7 +3558,7 @@ void xdp_do_flush_map(void) ...@@ -3558,7 +3558,7 @@ void xdp_do_flush_map(void)
__dev_map_flush(); __dev_map_flush();
break; break;
case BPF_MAP_TYPE_CPUMAP: case BPF_MAP_TYPE_CPUMAP:
__cpu_map_flush(map); __cpu_map_flush();
break; break;
case BPF_MAP_TYPE_XSKMAP: case BPF_MAP_TYPE_XSKMAP:
__xsk_map_flush(); __xsk_map_flush();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment