Skip to content

L
linux
Commit f836a56e authored by KP Singh's avatar KP Singh Committed by Alexei Starovoitov
Browse files
Options

bpf: Generalize bpf_sk_storage 

Refactor the functionality in bpf_sk_storage.c so that concept of
storage linked to kernel objects can be extended to other objects like
inode, task_struct etc.

Each new local storage will still be a separate map and provide its own
set of helpers. This allows for future object specific extensions and
still share a lot of the underlying implementation.

This includes the changes suggested by Martin in:

  https://lore.kernel.org/bpf/20200725013047.4006241-1-kafai@fb.com/

adding new map operations to support bpf_local_storage maps:

* storages for different kernel objects to optionally have different
  memory charging strategy (map_local_storage_charge,
  map_local_storage_uncharge)
* Functionality to extract the storage pointer from a pointer to the
  owning object (map_owner_storage_ptr)
Co-developed-by: default avatarMartin KaFai Lau <kafai@fb.com>
Signed-off-by: default avatarMartin KaFai Lau <kafai@fb.com>
Signed-off-by: default avatarKP Singh <kpsingh@google.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20200825182919.1118197-4-kpsingh@chromium.org
parent 4cc9ce4e
Show whitespace changes
Inline Side-by-side
Showing with 228 additions and 86 deletions
include/linux/bpf.h
View file @ f836a56e
...@@ -34,6 +34,8 @@ struct btf_type; ...@@ -34,6 +34,8 @@ struct btf_type;
struct exception_table_entry; struct exception_table_entry;
struct seq_operations; struct seq_operations;
struct bpf_iter_aux_info; struct bpf_iter_aux_info;
struct bpf_local_storage;
struct bpf_local_storage_map;
extern struct idr btf_idr; extern struct idr btf_idr;
extern spinlock_t btf_idr_lock; extern spinlock_t btf_idr_lock;
...@@ -104,6 +106,12 @@ struct bpf_map_ops { ...@@ -104,6 +106,12 @@ struct bpf_map_ops {
__poll_t (*map_poll)(struct bpf_map *map, struct file *filp, __poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
struct poll_table_struct *pts); struct poll_table_struct *pts);
/* Functions called by bpf_local_storage maps */
int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,
void *owner, u32 size);
void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,
void *owner, u32 size);
struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);
/* BTF name and id of struct allocated by map_alloc */ /* BTF name and id of struct allocated by map_alloc */
const char * const map_btf_name; const char * const map_btf_name;
int *map_btf_id; int *map_btf_id;
......
include/net/bpf_sk_storage.h
View file @ f836a56e
...@@ -3,8 +3,15 @@ ...@@ -3,8 +3,15 @@
#ifndef _BPF_SK_STORAGE_H #ifndef _BPF_SK_STORAGE_H
#define _BPF_SK_STORAGE_H #define _BPF_SK_STORAGE_H
#include <linux/rculist.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/spinlock.h> #include <linux/spinlock.h>
#include <linux/bpf.h>
#include <net/sock.h>
#include <uapi/linux/sock_diag.h>
#include <uapi/linux/btf.h>
struct sock; struct sock;
...@@ -13,6 +20,7 @@ void bpf_sk_storage_free(struct sock *sk); ...@@ -13,6 +20,7 @@ void bpf_sk_storage_free(struct sock *sk);
extern const struct bpf_func_proto bpf_sk_storage_get_proto; extern const struct bpf_func_proto bpf_sk_storage_get_proto;
extern const struct bpf_func_proto bpf_sk_storage_delete_proto; extern const struct bpf_func_proto bpf_sk_storage_delete_proto;
struct bpf_local_storage_elem;
struct bpf_sk_storage_diag; struct bpf_sk_storage_diag;
struct sk_buff; struct sk_buff;
struct nlattr; struct nlattr;
...@@ -34,6 +42,50 @@ u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache); ...@@ -34,6 +42,50 @@ u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache);
void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache, void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
u16 idx); u16 idx);
/* Helper functions for bpf_local_storage */
int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr);
struct bpf_local_storage_data *
bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
struct bpf_local_storage_map *smap,
bool cacheit_lockit);
void bpf_local_storage_map_free(struct bpf_local_storage_map *smap);
int bpf_local_storage_map_check_btf(const struct bpf_map *map,
const struct btf *btf,
const struct btf_type *key_type,
const struct btf_type *value_type);
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
struct bpf_local_storage_elem *selem);
bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
struct bpf_local_storage_elem *selem,
bool uncharge_omem);
void bpf_selem_unlink(struct bpf_local_storage_elem *selem);
void bpf_selem_link_map(struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *selem);
void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
struct bpf_local_storage_elem *
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
bool charge_mem);
int
bpf_local_storage_alloc(void *owner,
struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *first_selem);
struct bpf_local_storage_data *
bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
void *value, u64 map_flags);
#ifdef CONFIG_BPF_SYSCALL #ifdef CONFIG_BPF_SYSCALL
int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk); int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk);
struct bpf_sk_storage_diag * struct bpf_sk_storage_diag *
......
include/uapi/linux/bpf.h
View file @ f836a56e
...@@ -3765,9 +3765,13 @@ enum { ...@@ -3765,9 +3765,13 @@ enum {
BPF_F_SYSCTL_BASE_NAME = (1ULL << 0), BPF_F_SYSCTL_BASE_NAME = (1ULL << 0),
}; };
/* BPF_FUNC_sk_storage_get flags */ /* BPF_FUNC_<kernel_obj>_storage_get flags */
enum { enum {
BPF_SK_STORAGE_GET_F_CREATE = (1ULL << 0), BPF_LOCAL_STORAGE_GET_F_CREATE = (1ULL << 0),
/* BPF_SK_STORAGE_GET_F_CREATE is only kept for backward compatibility
* and BPF_LOCAL_STORAGE_GET_F_CREATE must be used instead.
*/
BPF_SK_STORAGE_GET_F_CREATE = BPF_LOCAL_STORAGE_GET_F_CREATE,
}; };
/* BPF_FUNC_read_branch_records flags. */ /* BPF_FUNC_read_branch_records flags. */
......
net/core/bpf_sk_storage.c
View file @ f836a56e
...@@ -84,7 +84,7 @@ struct bpf_local_storage_elem { ...@@ -84,7 +84,7 @@ struct bpf_local_storage_elem {
struct bpf_local_storage { struct bpf_local_storage {
struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE]; struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
struct hlist_head list; /* List of bpf_local_storage_elem */ struct hlist_head list; /* List of bpf_local_storage_elem */
struct sock *owner; /* The object that owns the above "list" of void *owner; /* The object that owns the above "list" of
* bpf_local_storage_elem. * bpf_local_storage_elem.
*/ */
struct rcu_head rcu; struct rcu_head rcu;
...@@ -110,6 +110,33 @@ static int omem_charge(struct sock *sk, unsigned int size) ...@@ -110,6 +110,33 @@ static int omem_charge(struct sock *sk, unsigned int size)
return -ENOMEM; return -ENOMEM;
} }
static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size)
{
struct bpf_map *map = &smap->map;
if (!map->ops->map_local_storage_charge)
return 0;
return map->ops->map_local_storage_charge(smap, owner, size);
}
static void mem_uncharge(struct bpf_local_storage_map *smap, void *owner,
u32 size)
{
struct bpf_map *map = &smap->map;
if (map->ops->map_local_storage_uncharge)
map->ops->map_local_storage_uncharge(smap, owner, size);
}
static struct bpf_local_storage __rcu **
owner_storage(struct bpf_local_storage_map *smap, void *owner)
{
struct bpf_map *map = &smap->map;
return map->ops->map_owner_storage_ptr(owner);
}
static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem) static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem)
{ {
return !hlist_unhashed(&selem->snode); return !hlist_unhashed(&selem->snode);
...@@ -120,13 +147,13 @@ static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem) ...@@ -120,13 +147,13 @@ static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)
return !hlist_unhashed(&selem->map_node); return !hlist_unhashed(&selem->map_node);
} }
static struct bpf_local_storage_elem * struct bpf_local_storage_elem *
bpf_selem_alloc(struct bpf_local_storage_map *smap, struct sock *sk, bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,
void *value, bool charge_omem) void *value, bool charge_mem)
{ {
struct bpf_local_storage_elem *selem; struct bpf_local_storage_elem *selem;
if (charge_omem && omem_charge(sk, smap->elem_size)) if (charge_mem && mem_charge(smap, owner, smap->elem_size))
return NULL; return NULL;
selem = kzalloc(smap->elem_size, GFP_ATOMIC | __GFP_NOWARN); selem = kzalloc(smap->elem_size, GFP_ATOMIC | __GFP_NOWARN);
...@@ -136,8 +163,8 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, struct sock *sk, ...@@ -136,8 +163,8 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, struct sock *sk,
return selem; return selem;
} }
if (charge_omem) if (charge_mem)
atomic_sub(smap->elem_size, &sk->sk_omem_alloc); mem_uncharge(smap, owner, smap->elem_size);
return NULL; return NULL;
} }
...@@ -146,32 +173,32 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, struct sock *sk, ...@@ -146,32 +173,32 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, struct sock *sk,
* The caller must ensure selem->smap is still valid to be * The caller must ensure selem->smap is still valid to be
* dereferenced for its smap->elem_size and smap->cache_idx. * dereferenced for its smap->elem_size and smap->cache_idx.
*/ */
static bool bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
struct bpf_local_storage_elem *selem, struct bpf_local_storage_elem *selem,
bool uncharge_omem) bool uncharge_mem)
{ {
struct bpf_local_storage_map *smap; struct bpf_local_storage_map *smap;
bool free_local_storage; bool free_local_storage;
struct sock *sk; void *owner;
smap = rcu_dereference(SDATA(selem)->smap); smap = rcu_dereference(SDATA(selem)->smap);
sk = local_storage->owner; owner = local_storage->owner;
/* All uncharging on the owner must be done first. /* All uncharging on the owner must be done first.
* The owner may be freed once the last selem is unlinked * The owner may be freed once the last selem is unlinked
* from local_storage. * from local_storage.
*/ */
if (uncharge_omem) if (uncharge_mem)
atomic_sub(smap->elem_size, &sk->sk_omem_alloc); mem_uncharge(smap, owner, smap->elem_size);
free_local_storage = hlist_is_singular_node(&selem->snode, free_local_storage = hlist_is_singular_node(&selem->snode,
&local_storage->list); &local_storage->list);
if (free_local_storage) { if (free_local_storage) {
atomic_sub(sizeof(struct bpf_local_storage), &sk->sk_omem_alloc); mem_uncharge(smap, owner, sizeof(struct bpf_local_storage));
local_storage->owner = NULL; local_storage->owner = NULL;
/* After this RCU_INIT, sk may be freed and cannot be used */
RCU_INIT_POINTER(sk->sk_bpf_storage, NULL); /* After this RCU_INIT, owner may be freed and cannot be used */
RCU_INIT_POINTER(*owner_storage(smap, owner), NULL);
/* local_storage is not freed now. local_storage->lock is /* local_storage is not freed now. local_storage->lock is
* still held and raw_spin_unlock_bh(&local_storage->lock) * still held and raw_spin_unlock_bh(&local_storage->lock)
...@@ -209,23 +236,22 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) ...@@ -209,23 +236,22 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem)
local_storage = rcu_dereference(selem->local_storage); local_storage = rcu_dereference(selem->local_storage);
raw_spin_lock_bh(&local_storage->lock); raw_spin_lock_bh(&local_storage->lock);
if (likely(selem_linked_to_storage(selem))) if (likely(selem_linked_to_storage(selem)))
free_local_storage = free_local_storage = bpf_selem_unlink_storage_nolock(
bpf_selem_unlink_storage_nolock(local_storage, selem, true); local_storage, selem, true);
raw_spin_unlock_bh(&local_storage->lock); raw_spin_unlock_bh(&local_storage->lock);
if (free_local_storage) if (free_local_storage)
kfree_rcu(local_storage, rcu); kfree_rcu(local_storage, rcu);
} }
static void void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
struct bpf_local_storage_elem *selem) struct bpf_local_storage_elem *selem)
{ {
RCU_INIT_POINTER(selem->local_storage, local_storage); RCU_INIT_POINTER(selem->local_storage, local_storage);
hlist_add_head(&selem->snode, &local_storage->list); hlist_add_head(&selem->snode, &local_storage->list);
} }
static void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)
{ {
struct bpf_local_storage_map *smap; struct bpf_local_storage_map *smap;
struct bpf_local_storage_map_bucket *b; struct bpf_local_storage_map_bucket *b;
...@@ -242,7 +268,7 @@ static void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) ...@@ -242,7 +268,7 @@ static void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)
raw_spin_unlock_bh(&b->lock); raw_spin_unlock_bh(&b->lock);
} }
static void bpf_selem_link_map(struct bpf_local_storage_map *smap, void bpf_selem_link_map(struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *selem) struct bpf_local_storage_elem *selem)
{ {
struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem); struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);
...@@ -253,7 +279,7 @@ static void bpf_selem_link_map(struct bpf_local_storage_map *smap, ...@@ -253,7 +279,7 @@ static void bpf_selem_link_map(struct bpf_local_storage_map *smap,
raw_spin_unlock_bh(&b->lock); raw_spin_unlock_bh(&b->lock);
} }
static void bpf_selem_unlink(struct bpf_local_storage_elem *selem) void bpf_selem_unlink(struct bpf_local_storage_elem *selem)
{ {
/* Always unlink from map before unlinking from local_storage /* Always unlink from map before unlinking from local_storage
* because selem will be freed after successfully unlinked from * because selem will be freed after successfully unlinked from
...@@ -263,7 +289,7 @@ static void bpf_selem_unlink(struct bpf_local_storage_elem *selem) ...@@ -263,7 +289,7 @@ static void bpf_selem_unlink(struct bpf_local_storage_elem *selem)
__bpf_selem_unlink_storage(selem); __bpf_selem_unlink_storage(selem);
} }
static struct bpf_local_storage_data * struct bpf_local_storage_data *
bpf_local_storage_lookup(struct bpf_local_storage *local_storage, bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
struct bpf_local_storage_map *smap, struct bpf_local_storage_map *smap,
bool cacheit_lockit) bool cacheit_lockit)
...@@ -329,40 +355,45 @@ static int check_flags(const struct bpf_local_storage_data *old_sdata, ...@@ -329,40 +355,45 @@ static int check_flags(const struct bpf_local_storage_data *old_sdata,
return 0; return 0;
} }
static int sk_storage_alloc(struct sock *sk, int bpf_local_storage_alloc(void *owner,
struct bpf_local_storage_map *smap, struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *first_selem) struct bpf_local_storage_elem *first_selem)
{ {
struct bpf_local_storage *prev_sk_storage, *sk_storage; struct bpf_local_storage *prev_storage, *storage;
struct bpf_local_storage **owner_storage_ptr;
int err; int err;
err = omem_charge(sk, sizeof(*sk_storage)); err = mem_charge(smap, owner, sizeof(*storage));
if (err) if (err)
return err; return err;
sk_storage = kzalloc(sizeof(*sk_storage), GFP_ATOMIC | __GFP_NOWARN); storage = kzalloc(sizeof(*storage), GFP_ATOMIC | __GFP_NOWARN);
if (!sk_storage) { if (!storage) {
err = -ENOMEM; err = -ENOMEM;
goto uncharge; goto uncharge;
} }
INIT_HLIST_HEAD(&sk_storage->list);
raw_spin_lock_init(&sk_storage->lock);
sk_storage->owner = sk;
bpf_selem_link_storage_nolock(sk_storage, first_selem); INIT_HLIST_HEAD(&storage->list);
raw_spin_lock_init(&storage->lock);
storage->owner = owner;
bpf_selem_link_storage_nolock(storage, first_selem);
bpf_selem_link_map(smap, first_selem); bpf_selem_link_map(smap, first_selem);
/* Publish sk_storage to sk. sk->sk_lock cannot be acquired.
* Hence, atomic ops is used to set sk->sk_bpf_storage owner_storage_ptr =
* from NULL to the newly allocated sk_storage ptr. (struct bpf_local_storage **)owner_storage(smap, owner);
/* Publish storage to the owner.
* Instead of using any lock of the kernel object (i.e. owner),
* cmpxchg will work with any kernel object regardless what
* the running context is, bh, irq...etc.
* *
* From now on, the sk->sk_bpf_storage pointer is protected * From now on, the owner->storage pointer (e.g. sk->sk_bpf_storage)
* by the sk_storage->lock. Hence, when freeing * is protected by the storage->lock. Hence, when freeing
* the sk->sk_bpf_storage, the sk_storage->lock must * the owner->storage, the storage->lock must be held before
* be held before setting sk->sk_bpf_storage to NULL. * setting owner->storage ptr to NULL.
*/ */
prev_sk_storage = cmpxchg((struct bpf_local_storage **)&sk->sk_bpf_storage, prev_storage = cmpxchg(owner_storage_ptr, NULL, storage);
NULL, sk_storage); if (unlikely(prev_storage)) {
if (unlikely(prev_sk_storage)) {
bpf_selem_unlink_map(first_selem); bpf_selem_unlink_map(first_selem);
err = -EAGAIN; err = -EAGAIN;
goto uncharge; goto uncharge;
...@@ -380,8 +411,8 @@ static int sk_storage_alloc(struct sock *sk, ...@@ -380,8 +411,8 @@ static int sk_storage_alloc(struct sock *sk,
return 0; return 0;
uncharge: uncharge:
kfree(sk_storage); kfree(storage);
atomic_sub(sizeof(*sk_storage), &sk->sk_omem_alloc); mem_uncharge(smap, owner, sizeof(*storage));
return err; return err;
} }
...@@ -390,38 +421,37 @@ static int sk_storage_alloc(struct sock *sk, ...@@ -390,38 +421,37 @@ static int sk_storage_alloc(struct sock *sk,
* Otherwise, it will become a leak (and other memory issues * Otherwise, it will become a leak (and other memory issues
* during map destruction). * during map destruction).
*/ */
static struct bpf_local_storage_data * struct bpf_local_storage_data *
bpf_local_storage_update(struct sock *sk, struct bpf_map *map, void *value, bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
u64 map_flags) void *value, u64 map_flags)
{ {
struct bpf_local_storage_data *old_sdata = NULL; struct bpf_local_storage_data *old_sdata = NULL;
struct bpf_local_storage_elem *selem; struct bpf_local_storage_elem *selem;
struct bpf_local_storage *local_storage; struct bpf_local_storage *local_storage;
struct bpf_local_storage_map *smap;
int err; int err;
/* BPF_EXIST and BPF_NOEXIST cannot be both set */ /* BPF_EXIST and BPF_NOEXIST cannot be both set */
if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST) || if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST) ||
/* BPF_F_LOCK can only be used in a value with spin_lock */ /* BPF_F_LOCK can only be used in a value with spin_lock */
unlikely((map_flags & BPF_F_LOCK) && !map_value_has_spin_lock(map))) unlikely((map_flags & BPF_F_LOCK) &&
!map_value_has_spin_lock(&smap->map)))
return ERR_PTR(-EINVAL); return ERR_PTR(-EINVAL);
smap = (struct bpf_local_storage_map *)map; local_storage = rcu_dereference(*owner_storage(smap, owner));
local_storage = rcu_dereference(sk->sk_bpf_storage);
if (!local_storage || hlist_empty(&local_storage->list)) { if (!local_storage || hlist_empty(&local_storage->list)) {
/* Very first elem for the owner */ /* Very first elem for the owner */
err = check_flags(NULL, map_flags); err = check_flags(NULL, map_flags);
if (err) if (err)
return ERR_PTR(err); return ERR_PTR(err);
selem = bpf_selem_alloc(smap, sk, value, true); selem = bpf_selem_alloc(smap, owner, value, true);
if (!selem) if (!selem)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
err = sk_storage_alloc(sk, smap, selem); err = bpf_local_storage_alloc(owner, smap, selem);
if (err) { if (err) {
kfree(selem); kfree(selem);
atomic_sub(smap->elem_size, &sk->sk_omem_alloc); mem_uncharge(smap, owner, smap->elem_size);
return ERR_PTR(err); return ERR_PTR(err);
} }
...@@ -439,7 +469,7 @@ bpf_local_storage_update(struct sock *sk, struct bpf_map *map, void *value, ...@@ -439,7 +469,7 @@ bpf_local_storage_update(struct sock *sk, struct bpf_map *map, void *value,
if (err) if (err)
return ERR_PTR(err); return ERR_PTR(err);
if (old_sdata && selem_linked_to_storage(SELEM(old_sdata))) { if (old_sdata && selem_linked_to_storage(SELEM(old_sdata))) {
copy_map_value_locked(map, old_sdata->data, copy_map_value_locked(&smap->map, old_sdata->data,
value, false); value, false);
return old_sdata; return old_sdata;
} }
...@@ -464,7 +494,8 @@ bpf_local_storage_update(struct sock *sk, struct bpf_map *map, void *value, ...@@ -464,7 +494,8 @@ bpf_local_storage_update(struct sock *sk, struct bpf_map *map, void *value,
goto unlock_err; goto unlock_err;
if (old_sdata && (map_flags & BPF_F_LOCK)) { if (old_sdata && (map_flags & BPF_F_LOCK)) {
copy_map_value_locked(map, old_sdata->data, value, false); copy_map_value_locked(&smap->map, old_sdata->data, value,
false);
selem = SELEM(old_sdata); selem = SELEM(old_sdata);
goto unlock; goto unlock;
} }
...@@ -478,7 +509,7 @@ bpf_local_storage_update(struct sock *sk, struct bpf_map *map, void *value, ...@@ -478,7 +509,7 @@ bpf_local_storage_update(struct sock *sk, struct bpf_map *map, void *value,
* old_sdata will not be uncharged later during * old_sdata will not be uncharged later during
* bpf_selem_unlink_storage_nolock(). * bpf_selem_unlink_storage_nolock().
*/ */
selem = bpf_selem_alloc(smap, sk, value, !old_sdata); selem = bpf_selem_alloc(smap, owner, value, !old_sdata);
if (!selem) { if (!selem) {
err = -ENOMEM; err = -ENOMEM;
goto unlock_err; goto unlock_err;
...@@ -591,17 +622,12 @@ void bpf_sk_storage_free(struct sock *sk) ...@@ -591,17 +622,12 @@ void bpf_sk_storage_free(struct sock *sk)
kfree_rcu(sk_storage, rcu); kfree_rcu(sk_storage, rcu);
} }
static void bpf_local_storage_map_free(struct bpf_map *map) void bpf_local_storage_map_free(struct bpf_local_storage_map *smap)
{ {
struct bpf_local_storage_elem *selem; struct bpf_local_storage_elem *selem;
struct bpf_local_storage_map *smap;
struct bpf_local_storage_map_bucket *b; struct bpf_local_storage_map_bucket *b;
unsigned int i; unsigned int i;
smap = (struct bpf_local_storage_map *)map;
bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx);
/* Note that this map might be concurrently cloned from /* Note that this map might be concurrently cloned from
* bpf_sk_storage_clone. Wait for any existing bpf_sk_storage_clone * bpf_sk_storage_clone. Wait for any existing bpf_sk_storage_clone
* RCU read section to finish before proceeding. New RCU * RCU read section to finish before proceeding. New RCU
...@@ -646,7 +672,16 @@ static void bpf_local_storage_map_free(struct bpf_map *map) ...@@ -646,7 +672,16 @@ static void bpf_local_storage_map_free(struct bpf_map *map)
synchronize_rcu(); synchronize_rcu();
kvfree(smap->buckets); kvfree(smap->buckets);
kfree(map); kfree(smap);
}
static void sk_storage_map_free(struct bpf_map *map)
{
struct bpf_local_storage_map *smap;
smap = (struct bpf_local_storage_map *)map;
bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx);
bpf_local_storage_map_free(smap);
} }
/* U16_MAX is much more than enough for sk local storage /* U16_MAX is much more than enough for sk local storage
...@@ -658,7 +693,7 @@ static void bpf_local_storage_map_free(struct bpf_map *map) ...@@ -658,7 +693,7 @@ static void bpf_local_storage_map_free(struct bpf_map *map)
sizeof(struct bpf_local_storage_elem)), \ sizeof(struct bpf_local_storage_elem)), \
(U16_MAX - sizeof(struct bpf_local_storage_elem))) (U16_MAX - sizeof(struct bpf_local_storage_elem)))
static int bpf_local_storage_map_alloc_check(union bpf_attr *attr) int bpf_local_storage_map_alloc_check(union bpf_attr *attr)
{ {
if (attr->map_flags & ~BPF_LOCAL_STORAGE_CREATE_FLAG_MASK || if (attr->map_flags & ~BPF_LOCAL_STORAGE_CREATE_FLAG_MASK ||
!(attr->map_flags & BPF_F_NO_PREALLOC) || !(attr->map_flags & BPF_F_NO_PREALLOC) ||
...@@ -677,7 +712,7 @@ static int bpf_local_storage_map_alloc_check(union bpf_attr *attr) ...@@ -677,7 +712,7 @@ static int bpf_local_storage_map_alloc_check(union bpf_attr *attr)
return 0; return 0;
} }
static struct bpf_map *bpf_local_storage_map_alloc(union bpf_attr *attr) struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr)
{ {
struct bpf_local_storage_map *smap; struct bpf_local_storage_map *smap;
unsigned int i; unsigned int i;
...@@ -717,8 +752,19 @@ static struct bpf_map *bpf_local_storage_map_alloc(union bpf_attr *attr) ...@@ -717,8 +752,19 @@ static struct bpf_map *bpf_local_storage_map_alloc(union bpf_attr *attr)
smap->elem_size = smap->elem_size =
sizeof(struct bpf_local_storage_elem) + attr->value_size; sizeof(struct bpf_local_storage_elem) + attr->value_size;
smap->cache_idx = bpf_local_storage_cache_idx_get(&sk_cache);
return smap;
}
static struct bpf_map *sk_storage_map_alloc(union bpf_attr *attr)
{
struct bpf_local_storage_map *smap;
smap = bpf_local_storage_map_alloc(attr);
if (IS_ERR(smap))
return ERR_CAST(smap);
smap->cache_idx = bpf_local_storage_cache_idx_get(&sk_cache);
return &smap->map; return &smap->map;
} }
...@@ -728,7 +774,7 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, ...@@ -728,7 +774,7 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key,
return -ENOTSUPP; return -ENOTSUPP;
} }
static int bpf_local_storage_map_check_btf(const struct bpf_map *map, int bpf_local_storage_map_check_btf(const struct bpf_map *map,
const struct btf *btf, const struct btf *btf,
const struct btf_type *key_type, const struct btf_type *key_type,
const struct btf_type *value_type) const struct btf_type *value_type)
...@@ -772,7 +818,8 @@ static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key, ...@@ -772,7 +818,8 @@ static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
fd = *(int *)key; fd = *(int *)key;
sock = sockfd_lookup(fd, &err); sock = sockfd_lookup(fd, &err);
if (sock) { if (sock) {
sdata = bpf_local_storage_update(sock->sk, map, value, sdata = bpf_local_storage_update(
sock->sk, (struct bpf_local_storage_map *)map, value,
map_flags); map_flags);
sockfd_put(sock); sockfd_put(sock);
return PTR_ERR_OR_ZERO(sdata); return PTR_ERR_OR_ZERO(sdata);
...@@ -862,7 +909,7 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) ...@@ -862,7 +909,7 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
bpf_selem_link_map(smap, copy_selem); bpf_selem_link_map(smap, copy_selem);
bpf_selem_link_storage_nolock(new_sk_storage, copy_selem); bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
} else { } else {
ret = sk_storage_alloc(newsk, smap, copy_selem); ret = bpf_local_storage_alloc(newsk, smap, copy_selem);
if (ret) { if (ret) {
kfree(copy_selem); kfree(copy_selem);
atomic_sub(smap->elem_size, atomic_sub(smap->elem_size,
...@@ -906,7 +953,9 @@ BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, ...@@ -906,7 +953,9 @@ BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
* destruction). * destruction).
*/ */
refcount_inc_not_zero(&sk->sk_refcnt)) { refcount_inc_not_zero(&sk->sk_refcnt)) {
sdata = bpf_local_storage_update(sk, map, value, BPF_NOEXIST); sdata = bpf_local_storage_update(
sk, (struct bpf_local_storage_map *)map, value,
BPF_NOEXIST);
/* sk must be a fullsock (guaranteed by verifier), /* sk must be a fullsock (guaranteed by verifier),
* so sock_gen_put() is unnecessary. * so sock_gen_put() is unnecessary.
*/ */
...@@ -931,11 +980,33 @@ BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk) ...@@ -931,11 +980,33 @@ BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
return -ENOENT; return -ENOENT;
} }
static int sk_storage_charge(struct bpf_local_storage_map *smap,
void *owner, u32 size)
{
return omem_charge(owner, size);
}
static void sk_storage_uncharge(struct bpf_local_storage_map *smap,
void *owner, u32 size)
{
struct sock *sk = owner;
atomic_sub(size, &sk->sk_omem_alloc);
}
static struct bpf_local_storage __rcu **
sk_storage_ptr(void *owner)
{
struct sock *sk = owner;
return &sk->sk_bpf_storage;
}
static int sk_storage_map_btf_id; static int sk_storage_map_btf_id;
const struct bpf_map_ops sk_storage_map_ops = { const struct bpf_map_ops sk_storage_map_ops = {
.map_alloc_check = bpf_local_storage_map_alloc_check, .map_alloc_check = bpf_local_storage_map_alloc_check,
.map_alloc = bpf_local_storage_map_alloc, .map_alloc = sk_storage_map_alloc,
.map_free = bpf_local_storage_map_free, .map_free = sk_storage_map_free,
.map_get_next_key = notsupp_get_next_key, .map_get_next_key = notsupp_get_next_key,
.map_lookup_elem = bpf_fd_sk_storage_lookup_elem, .map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
.map_update_elem = bpf_fd_sk_storage_update_elem, .map_update_elem = bpf_fd_sk_storage_update_elem,
...@@ -943,6 +1014,9 @@ const struct bpf_map_ops sk_storage_map_ops = { ...@@ -943,6 +1014,9 @@ const struct bpf_map_ops sk_storage_map_ops = {
.map_check_btf = bpf_local_storage_map_check_btf, .map_check_btf = bpf_local_storage_map_check_btf,
.map_btf_name = "bpf_local_storage_map", .map_btf_name = "bpf_local_storage_map",
.map_btf_id = &sk_storage_map_btf_id, .map_btf_id = &sk_storage_map_btf_id,
.map_local_storage_charge = sk_storage_charge,
.map_local_storage_uncharge = sk_storage_uncharge,
.map_owner_storage_ptr = sk_storage_ptr,
}; };
const struct bpf_func_proto bpf_sk_storage_get_proto = { const struct bpf_func_proto bpf_sk_storage_get_proto = {
......
tools/include/uapi/linux/bpf.h
View file @ f836a56e
...@@ -3765,9 +3765,13 @@ enum { ...@@ -3765,9 +3765,13 @@ enum {
BPF_F_SYSCTL_BASE_NAME = (1ULL << 0), BPF_F_SYSCTL_BASE_NAME = (1ULL << 0),
}; };
/* BPF_FUNC_sk_storage_get flags */ /* BPF_FUNC_<kernel_obj>_storage_get flags */
enum { enum {
BPF_SK_STORAGE_GET_F_CREATE = (1ULL << 0), BPF_LOCAL_STORAGE_GET_F_CREATE = (1ULL << 0),
/* BPF_SK_STORAGE_GET_F_CREATE is only kept for backward compatibility
* and BPF_LOCAL_STORAGE_GET_F_CREATE must be used instead.
*/
BPF_SK_STORAGE_GET_F_CREATE = BPF_LOCAL_STORAGE_GET_F_CREATE,
}; };
/* BPF_FUNC_read_branch_records flags. */ /* BPF_FUNC_read_branch_records flags. */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7