Commit ca5d1a7f authored by Alexei Starovoitov's avatar Alexei Starovoitov

Merge branch 'bpf_line_info'

Martin Lau says:

====================
This patch series introduces the bpf_line_info.  Please see individual patch
for details.

It will be useful for introspection purpose, like:

[root@arch-fb-vm1 bpf]# ~/devshare/fb-kernel/linux/tools/bpf/bpftool/bpftool prog dump jited pinned /sys/fs/bpf/test_btf_haskv
[...]
int test_long_fname_2(struct dummy_tracepoint_args * arg):
bpf_prog_44a040bf25481309_test_long_fname_2:
; static int test_long_fname_2(struct dummy_tracepoint_args *arg)
   0:   push   %rbp
   1:   mov    %rsp,%rbp
   4:   sub    $0x30,%rsp
   b:   sub    $0x28,%rbp
   f:   mov    %rbx,0x0(%rbp)
  13:   mov    %r13,0x8(%rbp)
  17:   mov    %r14,0x10(%rbp)
  1b:   mov    %r15,0x18(%rbp)
  1f:   xor    %eax,%eax
  21:   mov    %rax,0x20(%rbp)
  25:   xor    %esi,%esi
; int key = 0;
  27:   mov    %esi,-0x4(%rbp)
; if (!arg->sock)
  2a:   mov    0x8(%rdi),%rdi
; if (!arg->sock)
  2e:   cmp    $0x0,%rdi
  32:   je     0x0000000000000070
  34:   mov    %rbp,%rsi
; counts = bpf_map_lookup_elem(&btf_map, &key);
  37:   add    $0xfffffffffffffffc,%rsi
  3b:   movabs $0xffff8881139d7480,%rdi
  45:   add    $0x110,%rdi
  4c:   mov    0x0(%rsi),%eax
  4f:   cmp    $0x4,%rax
  53:   jae    0x000000000000005e
  55:   shl    $0x3,%rax
  59:   add    %rdi,%rax
  5c:   jmp    0x0000000000000060
  5e:   xor    %eax,%eax
; if (!counts)
  60:   cmp    $0x0,%rax
  64:   je     0x0000000000000070
; counts->v6++;
  66:   mov    0x4(%rax),%edi
  69:   add    $0x1,%rdi
  6d:   mov    %edi,0x4(%rax)
  70:   mov    0x0(%rbp),%rbx
  74:   mov    0x8(%rbp),%r13
  78:   mov    0x10(%rbp),%r14
  7c:   mov    0x18(%rbp),%r15
  80:   add    $0x28,%rbp
  84:   leaveq
  85:   retq
[...]
====================
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 6baefa1a b053b439
...@@ -1181,6 +1181,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ...@@ -1181,6 +1181,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
} }
if (!image || !prog->is_func || extra_pass) { if (!image || !prog->is_func || extra_pass) {
if (image)
bpf_prog_fill_jited_linfo(prog, addrs);
out_addrs: out_addrs:
kfree(addrs); kfree(addrs);
kfree(jit_data); kfree(jit_data);
......
...@@ -319,7 +319,28 @@ struct bpf_prog_aux { ...@@ -319,7 +319,28 @@ struct bpf_prog_aux {
struct bpf_prog_offload *offload; struct bpf_prog_offload *offload;
struct btf *btf; struct btf *btf;
struct bpf_func_info *func_info; struct bpf_func_info *func_info;
/* bpf_line_info loaded from userspace. linfo->insn_off
* has the xlated insn offset.
* Both the main and sub prog share the same linfo.
* The subprog can access its first linfo by
* using the linfo_idx.
*/
struct bpf_line_info *linfo;
/* jited_linfo is the jited addr of the linfo. It has a
* one to one mapping to linfo:
* jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
* Both the main and sub prog share the same jited_linfo.
* The subprog can access its first jited_linfo by
* using the linfo_idx.
*/
void **jited_linfo;
u32 func_info_cnt; u32 func_info_cnt;
u32 nr_linfo;
/* subprog can use linfo_idx to access its first linfo and
* jited_linfo.
* main prog always has linfo_idx == 0
*/
u32 linfo_idx;
union { union {
struct work_struct work; struct work_struct work;
struct rcu_head rcu; struct rcu_head rcu;
......
...@@ -203,6 +203,7 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) ...@@ -203,6 +203,7 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
struct bpf_subprog_info { struct bpf_subprog_info {
u32 start; /* insn idx of function entry point */ u32 start; /* insn idx of function entry point */
u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
u16 stack_depth; /* max. stack depth used by this function */ u16 stack_depth; /* max. stack depth used by this function */
}; };
......
...@@ -46,6 +46,7 @@ void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, ...@@ -46,6 +46,7 @@ void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
struct seq_file *m); struct seq_file *m);
int btf_get_fd_by_id(u32 id); int btf_get_fd_by_id(u32 id);
u32 btf_id(const struct btf *btf); u32 btf_id(const struct btf *btf);
bool btf_name_offset_valid(const struct btf *btf, u32 offset);
#ifdef CONFIG_BPF_SYSCALL #ifdef CONFIG_BPF_SYSCALL
const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);
......
...@@ -718,6 +718,13 @@ void bpf_prog_free(struct bpf_prog *fp); ...@@ -718,6 +718,13 @@ void bpf_prog_free(struct bpf_prog *fp);
bool bpf_opcode_in_insntable(u8 code); bool bpf_opcode_in_insntable(u8 code);
void bpf_prog_free_linfo(struct bpf_prog *prog);
void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
const u32 *insn_to_jit_off);
int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog);
void bpf_prog_free_jited_linfo(struct bpf_prog *prog);
void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);
struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags); struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
gfp_t gfp_extra_flags); gfp_t gfp_extra_flags);
......
...@@ -356,6 +356,9 @@ union bpf_attr { ...@@ -356,6 +356,9 @@ union bpf_attr {
__u32 func_info_rec_size; /* userspace bpf_func_info size */ __u32 func_info_rec_size; /* userspace bpf_func_info size */
__aligned_u64 func_info; /* func info */ __aligned_u64 func_info; /* func info */
__u32 func_info_cnt; /* number of bpf_func_info records */ __u32 func_info_cnt; /* number of bpf_func_info records */
__u32 line_info_rec_size; /* userspace bpf_line_info size */
__aligned_u64 line_info; /* line info */
__u32 line_info_cnt; /* number of bpf_line_info records */
}; };
struct { /* anonymous struct used by BPF_OBJ_* commands */ struct { /* anonymous struct used by BPF_OBJ_* commands */
...@@ -2679,6 +2682,12 @@ struct bpf_prog_info { ...@@ -2679,6 +2682,12 @@ struct bpf_prog_info {
__u32 func_info_rec_size; __u32 func_info_rec_size;
__aligned_u64 func_info; __aligned_u64 func_info;
__u32 func_info_cnt; __u32 func_info_cnt;
__u32 line_info_cnt;
__aligned_u64 line_info;
__aligned_u64 jited_line_info;
__u32 jited_line_info_cnt;
__u32 line_info_rec_size;
__u32 jited_line_info_rec_size;
} __attribute__((aligned(8))); } __attribute__((aligned(8)));
struct bpf_map_info { struct bpf_map_info {
...@@ -2995,4 +3004,14 @@ struct bpf_func_info { ...@@ -2995,4 +3004,14 @@ struct bpf_func_info {
__u32 type_id; __u32 type_id;
}; };
#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10)
#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff)
struct bpf_line_info {
__u32 insn_off;
__u32 file_name_off;
__u32 line_off;
__u32 line_col;
};
#endif /* _UAPI__LINUX_BPF_H__ */ #endif /* _UAPI__LINUX_BPF_H__ */
...@@ -444,7 +444,7 @@ static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t) ...@@ -444,7 +444,7 @@ static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
return kind_ops[BTF_INFO_KIND(t->info)]; return kind_ops[BTF_INFO_KIND(t->info)];
} }
static bool btf_name_offset_valid(const struct btf *btf, u32 offset) bool btf_name_offset_valid(const struct btf *btf, u32 offset)
{ {
return BTF_STR_OFFSET_VALID(offset) && return BTF_STR_OFFSET_VALID(offset) &&
offset < btf->hdr.str_len; offset < btf->hdr.str_len;
......
...@@ -105,6 +105,91 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) ...@@ -105,6 +105,91 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
} }
EXPORT_SYMBOL_GPL(bpf_prog_alloc); EXPORT_SYMBOL_GPL(bpf_prog_alloc);
int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)
{
if (!prog->aux->nr_linfo || !prog->jit_requested)
return 0;
prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,
sizeof(*prog->aux->jited_linfo),
GFP_KERNEL | __GFP_NOWARN);
if (!prog->aux->jited_linfo)
return -ENOMEM;
return 0;
}
void bpf_prog_free_jited_linfo(struct bpf_prog *prog)
{
kfree(prog->aux->jited_linfo);
prog->aux->jited_linfo = NULL;
}
void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog)
{
if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])
bpf_prog_free_jited_linfo(prog);
}
/* The jit engine is responsible to provide an array
* for insn_off to the jited_off mapping (insn_to_jit_off).
*
* The idx to this array is the insn_off. Hence, the insn_off
* here is relative to the prog itself instead of the main prog.
* This array has one entry for each xlated bpf insn.
*
* jited_off is the byte off to the last byte of the jited insn.
*
* Hence, with
* insn_start:
* The first bpf insn off of the prog. The insn off
* here is relative to the main prog.
* e.g. if prog is a subprog, insn_start > 0
* linfo_idx:
* The prog's idx to prog->aux->linfo and jited_linfo
*
* jited_linfo[linfo_idx] = prog->bpf_func
*
* For i > linfo_idx,
*
* jited_linfo[i] = prog->bpf_func +
* insn_to_jit_off[linfo[i].insn_off - insn_start - 1]
*/
void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
const u32 *insn_to_jit_off)
{
u32 linfo_idx, insn_start, insn_end, nr_linfo, i;
const struct bpf_line_info *linfo;
void **jited_linfo;
if (!prog->aux->jited_linfo)
/* Userspace did not provide linfo */
return;
linfo_idx = prog->aux->linfo_idx;
linfo = &prog->aux->linfo[linfo_idx];
insn_start = linfo[0].insn_off;
insn_end = insn_start + prog->len;
jited_linfo = &prog->aux->jited_linfo[linfo_idx];
jited_linfo[0] = prog->bpf_func;
nr_linfo = prog->aux->nr_linfo - linfo_idx;
for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++)
/* The verifier ensures that linfo[i].insn_off is
* strictly increasing
*/
jited_linfo[i] = prog->bpf_func +
insn_to_jit_off[linfo[i].insn_off - insn_start - 1];
}
void bpf_prog_free_linfo(struct bpf_prog *prog)
{
bpf_prog_free_jited_linfo(prog);
kvfree(prog->aux->linfo);
}
struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
gfp_t gfp_extra_flags) gfp_t gfp_extra_flags)
{ {
...@@ -294,6 +379,26 @@ static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta, ...@@ -294,6 +379,26 @@ static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta,
return ret; return ret;
} }
static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta)
{
struct bpf_line_info *linfo;
u32 i, nr_linfo;
nr_linfo = prog->aux->nr_linfo;
if (!nr_linfo || !delta)
return;
linfo = prog->aux->linfo;
for (i = 0; i < nr_linfo; i++)
if (off < linfo[i].insn_off)
break;
/* Push all off < linfo[i].insn_off by delta */
for (; i < nr_linfo; i++)
linfo[i].insn_off += delta;
}
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
const struct bpf_insn *patch, u32 len) const struct bpf_insn *patch, u32 len)
{ {
...@@ -349,6 +454,8 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, ...@@ -349,6 +454,8 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
*/ */
BUG_ON(bpf_adj_branches(prog_adj, off, insn_delta, false)); BUG_ON(bpf_adj_branches(prog_adj, off, insn_delta, false));
bpf_adj_linfo(prog_adj, off, insn_delta);
return prog_adj; return prog_adj;
} }
...@@ -1591,13 +1698,20 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) ...@@ -1591,13 +1698,20 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
* be JITed, but falls back to the interpreter. * be JITed, but falls back to the interpreter.
*/ */
if (!bpf_prog_is_dev_bound(fp->aux)) { if (!bpf_prog_is_dev_bound(fp->aux)) {
*err = bpf_prog_alloc_jited_linfo(fp);
if (*err)
return fp;
fp = bpf_int_jit_compile(fp); fp = bpf_int_jit_compile(fp);
#ifdef CONFIG_BPF_JIT_ALWAYS_ON
if (!fp->jited) { if (!fp->jited) {
bpf_prog_free_jited_linfo(fp);
#ifdef CONFIG_BPF_JIT_ALWAYS_ON
*err = -ENOTSUPP; *err = -ENOTSUPP;
return fp; return fp;
}
#endif #endif
} else {
bpf_prog_free_unused_jited_linfo(fp);
}
} else { } else {
*err = bpf_prog_offload_compile(fp); *err = bpf_prog_offload_compile(fp);
if (*err) if (*err)
......
...@@ -1215,6 +1215,7 @@ static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) ...@@ -1215,6 +1215,7 @@ static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
bpf_prog_kallsyms_del_all(prog); bpf_prog_kallsyms_del_all(prog);
btf_put(prog->aux->btf); btf_put(prog->aux->btf);
kvfree(prog->aux->func_info); kvfree(prog->aux->func_info);
bpf_prog_free_linfo(prog);
call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
} }
...@@ -1439,7 +1440,7 @@ bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, ...@@ -1439,7 +1440,7 @@ bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type,
} }
/* last field in 'union bpf_attr' used by this command */ /* last field in 'union bpf_attr' used by this command */
#define BPF_PROG_LOAD_LAST_FIELD func_info_cnt #define BPF_PROG_LOAD_LAST_FIELD line_info_cnt
static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
{ {
...@@ -1560,6 +1561,7 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) ...@@ -1560,6 +1561,7 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
return err; return err;
free_used_maps: free_used_maps:
bpf_prog_free_linfo(prog);
kvfree(prog->aux->func_info); kvfree(prog->aux->func_info);
btf_put(prog->aux->btf); btf_put(prog->aux->btf);
bpf_prog_kallsyms_del_subprogs(prog); bpf_prog_kallsyms_del_subprogs(prog);
...@@ -2041,6 +2043,37 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) ...@@ -2041,6 +2043,37 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog)
return insns; return insns;
} }
static int set_info_rec_size(struct bpf_prog_info *info)
{
/*
* Ensure info.*_rec_size is the same as kernel expected size
*
* or
*
* Only allow zero *_rec_size if both _rec_size and _cnt are
* zero. In this case, the kernel will set the expected
* _rec_size back to the info.
*/
if ((info->func_info_cnt || info->func_info_rec_size) &&
info->func_info_rec_size != sizeof(struct bpf_func_info))
return -EINVAL;
if ((info->line_info_cnt || info->line_info_rec_size) &&
info->line_info_rec_size != sizeof(struct bpf_line_info))
return -EINVAL;
if ((info->jited_line_info_cnt || info->jited_line_info_rec_size) &&
info->jited_line_info_rec_size != sizeof(__u64))
return -EINVAL;
info->func_info_rec_size = sizeof(struct bpf_func_info);
info->line_info_rec_size = sizeof(struct bpf_line_info);
info->jited_line_info_rec_size = sizeof(__u64);
return 0;
}
static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
const union bpf_attr *attr, const union bpf_attr *attr,
union bpf_attr __user *uattr) union bpf_attr __user *uattr)
...@@ -2083,11 +2116,9 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, ...@@ -2083,11 +2116,9 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
return -EFAULT; return -EFAULT;
} }
if ((info.func_info_cnt || info.func_info_rec_size) && err = set_info_rec_size(&info);
info.func_info_rec_size != sizeof(struct bpf_func_info)) if (err)
return -EINVAL; return err;
info.func_info_rec_size = sizeof(struct bpf_func_info);
if (!capable(CAP_SYS_ADMIN)) { if (!capable(CAP_SYS_ADMIN)) {
info.jited_prog_len = 0; info.jited_prog_len = 0;
...@@ -2095,6 +2126,8 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, ...@@ -2095,6 +2126,8 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
info.nr_jited_ksyms = 0; info.nr_jited_ksyms = 0;
info.nr_jited_func_lens = 0; info.nr_jited_func_lens = 0;
info.func_info_cnt = 0; info.func_info_cnt = 0;
info.line_info_cnt = 0;
info.jited_line_info_cnt = 0;
goto done; goto done;
} }
...@@ -2251,6 +2284,44 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, ...@@ -2251,6 +2284,44 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
} }
} }
ulen = info.line_info_cnt;
info.line_info_cnt = prog->aux->nr_linfo;
if (info.line_info_cnt && ulen) {
if (bpf_dump_raw_ok()) {
__u8 __user *user_linfo;
user_linfo = u64_to_user_ptr(info.line_info);
ulen = min_t(u32, info.line_info_cnt, ulen);
if (copy_to_user(user_linfo, prog->aux->linfo,
info.line_info_rec_size * ulen))
return -EFAULT;
} else {
info.line_info = 0;
}
}
ulen = info.jited_line_info_cnt;
if (prog->aux->jited_linfo)
info.jited_line_info_cnt = prog->aux->nr_linfo;
else
info.jited_line_info_cnt = 0;
if (info.jited_line_info_cnt && ulen) {
if (bpf_dump_raw_ok()) {
__u64 __user *user_linfo;
u32 i;
user_linfo = u64_to_user_ptr(info.jited_line_info);
ulen = min_t(u32, info.jited_line_info_cnt, ulen);
for (i = 0; i < ulen; i++) {
if (put_user((__u64)(long)prog->aux->jited_linfo[i],
&user_linfo[i]))
return -EFAULT;
}
} else {
info.jited_line_info = 0;
}
}
done: done:
if (copy_to_user(uinfo, &info, info_len) || if (copy_to_user(uinfo, &info, info_len) ||
put_user(info_len, &uattr->info.info_len)) put_user(info_len, &uattr->info.info_len))
......
...@@ -4640,15 +4640,17 @@ static int check_cfg(struct bpf_verifier_env *env) ...@@ -4640,15 +4640,17 @@ static int check_cfg(struct bpf_verifier_env *env)
#define MIN_BPF_FUNCINFO_SIZE 8 #define MIN_BPF_FUNCINFO_SIZE 8
#define MAX_FUNCINFO_REC_SIZE 252 #define MAX_FUNCINFO_REC_SIZE 252
static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, static int check_btf_func(struct bpf_verifier_env *env,
union bpf_attr *attr, union bpf_attr __user *uattr) const union bpf_attr *attr,
union bpf_attr __user *uattr)
{ {
u32 i, nfuncs, urec_size, min_size, prev_offset; u32 i, nfuncs, urec_size, min_size, prev_offset;
u32 krec_size = sizeof(struct bpf_func_info); u32 krec_size = sizeof(struct bpf_func_info);
struct bpf_func_info *krecord = NULL; struct bpf_func_info *krecord;
const struct btf_type *type; const struct btf_type *type;
struct bpf_prog *prog;
const struct btf *btf;
void __user *urecord; void __user *urecord;
struct btf *btf;
int ret = 0; int ret = 0;
nfuncs = attr->func_info_cnt; nfuncs = attr->func_info_cnt;
...@@ -4668,20 +4670,15 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, ...@@ -4668,20 +4670,15 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env,
return -EINVAL; return -EINVAL;
} }
btf = btf_get_by_fd(attr->prog_btf_fd); prog = env->prog;
if (IS_ERR(btf)) { btf = prog->aux->btf;
verbose(env, "unable to get btf from fd\n");
return PTR_ERR(btf);
}
urecord = u64_to_user_ptr(attr->func_info); urecord = u64_to_user_ptr(attr->func_info);
min_size = min_t(u32, krec_size, urec_size); min_size = min_t(u32, krec_size, urec_size);
krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
if (!krecord) { if (!krecord)
ret = -ENOMEM; return -ENOMEM;
goto free_btf;
}
for (i = 0; i < nfuncs; i++) { for (i = 0; i < nfuncs; i++) {
ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
...@@ -4694,12 +4691,12 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, ...@@ -4694,12 +4691,12 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env,
if (put_user(min_size, &uattr->func_info_rec_size)) if (put_user(min_size, &uattr->func_info_rec_size))
ret = -EFAULT; ret = -EFAULT;
} }
goto free_btf; goto err_free;
} }
if (copy_from_user(&krecord[i], urecord, min_size)) { if (copy_from_user(&krecord[i], urecord, min_size)) {
ret = -EFAULT; ret = -EFAULT;
goto free_btf; goto err_free;
} }
/* check insn_off */ /* check insn_off */
...@@ -4709,20 +4706,20 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, ...@@ -4709,20 +4706,20 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env,
"nonzero insn_off %u for the first func info record", "nonzero insn_off %u for the first func info record",
krecord[i].insn_off); krecord[i].insn_off);
ret = -EINVAL; ret = -EINVAL;
goto free_btf; goto err_free;
} }
} else if (krecord[i].insn_off <= prev_offset) { } else if (krecord[i].insn_off <= prev_offset) {
verbose(env, verbose(env,
"same or smaller insn offset (%u) than previous func info record (%u)", "same or smaller insn offset (%u) than previous func info record (%u)",
krecord[i].insn_off, prev_offset); krecord[i].insn_off, prev_offset);
ret = -EINVAL; ret = -EINVAL;
goto free_btf; goto err_free;
} }
if (env->subprog_info[i].start != krecord[i].insn_off) { if (env->subprog_info[i].start != krecord[i].insn_off) {
verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
ret = -EINVAL; ret = -EINVAL;
goto free_btf; goto err_free;
} }
/* check type_id */ /* check type_id */
...@@ -4731,20 +4728,18 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, ...@@ -4731,20 +4728,18 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env,
verbose(env, "invalid type id %d in func info", verbose(env, "invalid type id %d in func info",
krecord[i].type_id); krecord[i].type_id);
ret = -EINVAL; ret = -EINVAL;
goto free_btf; goto err_free;
} }
prev_offset = krecord[i].insn_off; prev_offset = krecord[i].insn_off;
urecord += urec_size; urecord += urec_size;
} }
prog->aux->btf = btf;
prog->aux->func_info = krecord; prog->aux->func_info = krecord;
prog->aux->func_info_cnt = nfuncs; prog->aux->func_info_cnt = nfuncs;
return 0; return 0;
free_btf: err_free:
btf_put(btf);
kvfree(krecord); kvfree(krecord);
return ret; return ret;
} }
...@@ -4760,6 +4755,150 @@ static void adjust_btf_func(struct bpf_verifier_env *env) ...@@ -4760,6 +4755,150 @@ static void adjust_btf_func(struct bpf_verifier_env *env)
env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start; env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start;
} }
#define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \
sizeof(((struct bpf_line_info *)(0))->line_col))
#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
static int check_btf_line(struct bpf_verifier_env *env,
const union bpf_attr *attr,
union bpf_attr __user *uattr)
{
u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
struct bpf_subprog_info *sub;
struct bpf_line_info *linfo;
struct bpf_prog *prog;
const struct btf *btf;
void __user *ulinfo;
int err;
nr_linfo = attr->line_info_cnt;
if (!nr_linfo)
return 0;
rec_size = attr->line_info_rec_size;
if (rec_size < MIN_BPF_LINEINFO_SIZE ||
rec_size > MAX_LINEINFO_REC_SIZE ||
rec_size & (sizeof(u32) - 1))
return -EINVAL;
/* Need to zero it in case the userspace may
* pass in a smaller bpf_line_info object.
*/
linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
GFP_KERNEL | __GFP_NOWARN);
if (!linfo)
return -ENOMEM;
prog = env->prog;
btf = prog->aux->btf;
s = 0;
sub = env->subprog_info;
ulinfo = u64_to_user_ptr(attr->line_info);
expected_size = sizeof(struct bpf_line_info);
ncopy = min_t(u32, expected_size, rec_size);
for (i = 0; i < nr_linfo; i++) {
err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
if (err) {
if (err == -E2BIG) {
verbose(env, "nonzero tailing record in line_info");
if (put_user(expected_size,
&uattr->line_info_rec_size))
err = -EFAULT;
}
goto err_free;
}
if (copy_from_user(&linfo[i], ulinfo, ncopy)) {
err = -EFAULT;
goto err_free;
}
/*
* Check insn_off to ensure
* 1) strictly increasing AND
* 2) bounded by prog->len
*
* The linfo[0].insn_off == 0 check logically falls into
* the later "missing bpf_line_info for func..." case
* because the first linfo[0].insn_off must be the
* first sub also and the first sub must have
* subprog_info[0].start == 0.
*/
if ((i && linfo[i].insn_off <= prev_offset) ||
linfo[i].insn_off >= prog->len) {
verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
i, linfo[i].insn_off, prev_offset,
prog->len);
err = -EINVAL;
goto err_free;
}
if (!btf_name_offset_valid(btf, linfo[i].line_off) ||
!btf_name_offset_valid(btf, linfo[i].file_name_off)) {
verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
err = -EINVAL;
goto err_free;
}
if (s != env->subprog_cnt) {
if (linfo[i].insn_off == sub[s].start) {
sub[s].linfo_idx = i;
s++;
} else if (sub[s].start < linfo[i].insn_off) {
verbose(env, "missing bpf_line_info for func#%u\n", s);
err = -EINVAL;
goto err_free;
}
}
prev_offset = linfo[i].insn_off;
ulinfo += rec_size;
}
if (s != env->subprog_cnt) {
verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
env->subprog_cnt - s, s);
err = -EINVAL;
goto err_free;
}
prog->aux->linfo = linfo;
prog->aux->nr_linfo = nr_linfo;
return 0;
err_free:
kvfree(linfo);
return err;
}
static int check_btf_info(struct bpf_verifier_env *env,
const union bpf_attr *attr,
union bpf_attr __user *uattr)
{
struct btf *btf;
int err;
if (!attr->func_info_cnt && !attr->line_info_cnt)
return 0;
btf = btf_get_by_fd(attr->prog_btf_fd);
if (IS_ERR(btf))
return PTR_ERR(btf);
env->prog->aux->btf = btf;
err = check_btf_func(env, attr, uattr);
if (err)
return err;
err = check_btf_line(env, attr, uattr);
if (err)
return err;
return 0;
}
/* check %cur's range satisfies %old's */ /* check %cur's range satisfies %old's */
static bool range_within(struct bpf_reg_state *old, static bool range_within(struct bpf_reg_state *old,
struct bpf_reg_state *cur) struct bpf_reg_state *cur)
...@@ -6004,7 +6143,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) ...@@ -6004,7 +6143,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
int i, j, subprog_start, subprog_end = 0, len, subprog; int i, j, subprog_start, subprog_end = 0, len, subprog;
struct bpf_insn *insn; struct bpf_insn *insn;
void *old_bpf_func; void *old_bpf_func;
int err = -ENOMEM; int err;
if (env->subprog_cnt <= 1) if (env->subprog_cnt <= 1)
return 0; return 0;
...@@ -6035,6 +6174,11 @@ static int jit_subprogs(struct bpf_verifier_env *env) ...@@ -6035,6 +6174,11 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->imm = 1; insn->imm = 1;
} }
err = bpf_prog_alloc_jited_linfo(prog);
if (err)
goto out_undo_insn;
err = -ENOMEM;
func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
if (!func) if (!func)
goto out_undo_insn; goto out_undo_insn;
...@@ -6065,6 +6209,10 @@ static int jit_subprogs(struct bpf_verifier_env *env) ...@@ -6065,6 +6209,10 @@ static int jit_subprogs(struct bpf_verifier_env *env)
func[i]->aux->name[0] = 'F'; func[i]->aux->name[0] = 'F';
func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
func[i]->jit_requested = 1; func[i]->jit_requested = 1;
func[i]->aux->linfo = prog->aux->linfo;
func[i]->aux->nr_linfo = prog->aux->nr_linfo;
func[i]->aux->jited_linfo = prog->aux->jited_linfo;
func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
func[i] = bpf_int_jit_compile(func[i]); func[i] = bpf_int_jit_compile(func[i]);
if (!func[i]->jited) { if (!func[i]->jited) {
err = -ENOTSUPP; err = -ENOTSUPP;
...@@ -6138,6 +6286,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) ...@@ -6138,6 +6286,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
prog->bpf_func = func[0]->bpf_func; prog->bpf_func = func[0]->bpf_func;
prog->aux->func = func; prog->aux->func = func;
prog->aux->func_cnt = env->subprog_cnt; prog->aux->func_cnt = env->subprog_cnt;
bpf_prog_free_unused_jited_linfo(prog);
return 0; return 0;
out_free: out_free:
for (i = 0; i < env->subprog_cnt; i++) for (i = 0; i < env->subprog_cnt; i++)
...@@ -6154,6 +6303,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) ...@@ -6154,6 +6303,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->off = 0; insn->off = 0;
insn->imm = env->insn_aux_data[i].call_imm; insn->imm = env->insn_aux_data[i].call_imm;
} }
bpf_prog_free_jited_linfo(prog);
return err; return err;
} }
...@@ -6526,7 +6676,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, ...@@ -6526,7 +6676,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
if (ret < 0) if (ret < 0)
goto skip_full_check; goto skip_full_check;
ret = check_btf_func(env->prog, env, attr, uattr); ret = check_btf_info(env, attr, uattr);
if (ret < 0) if (ret < 0)
goto skip_full_check; goto skip_full_check;
......
...@@ -22,8 +22,8 @@ MAP COMMANDS ...@@ -22,8 +22,8 @@ MAP COMMANDS
============= =============
| **bpftool** **prog { show | list }** [*PROG*] | **bpftool** **prog { show | list }** [*PROG*]
| **bpftool** **prog dump xlated** *PROG* [{**file** *FILE* | **opcodes** | **visual**}] | **bpftool** **prog dump xlated** *PROG* [{**file** *FILE* | **opcodes** | **visual** | **linum**}]
| **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes**}] | **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes** | **linum**}]
| **bpftool** **prog pin** *PROG* *FILE* | **bpftool** **prog pin** *PROG* *FILE*
| **bpftool** **prog { load | loadall }** *OBJ* *PATH* [**type** *TYPE*] [**map** {**idx** *IDX* | **name** *NAME*} *MAP*] [**dev** *NAME*] | **bpftool** **prog { load | loadall }** *OBJ* *PATH* [**type** *TYPE*] [**map** {**idx** *IDX* | **name** *NAME*} *MAP*] [**dev** *NAME*]
| **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*] | **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*]
...@@ -56,7 +56,7 @@ DESCRIPTION ...@@ -56,7 +56,7 @@ DESCRIPTION
Output will start with program ID followed by program type and Output will start with program ID followed by program type and
zero or more named attributes (depending on kernel version). zero or more named attributes (depending on kernel version).
**bpftool prog dump xlated** *PROG* [{ **file** *FILE* | **opcodes** | **visual** }] **bpftool prog dump xlated** *PROG* [{ **file** *FILE* | **opcodes** | **visual** | **linum** }]
Dump eBPF instructions of the program from the kernel. By Dump eBPF instructions of the program from the kernel. By
default, eBPF will be disassembled and printed to standard default, eBPF will be disassembled and printed to standard
output in human-readable format. In this case, **opcodes** output in human-readable format. In this case, **opcodes**
...@@ -69,13 +69,21 @@ DESCRIPTION ...@@ -69,13 +69,21 @@ DESCRIPTION
built instead, and eBPF instructions will be presented with built instead, and eBPF instructions will be presented with
CFG in DOT format, on standard output. CFG in DOT format, on standard output.
**bpftool prog dump jited** *PROG* [{ **file** *FILE* | **opcodes** }] If the prog has line_info available, the source line will
be displayed by default. If **linum** is specified,
the filename, line number and line column will also be
displayed on top of the source line.
**bpftool prog dump jited** *PROG* [{ **file** *FILE* | **opcodes** | **linum** }]
Dump jited image (host machine code) of the program. Dump jited image (host machine code) of the program.
If *FILE* is specified image will be written to a file, If *FILE* is specified image will be written to a file,
otherwise it will be disassembled and printed to stdout. otherwise it will be disassembled and printed to stdout.
**opcodes** controls if raw opcodes will be printed. **opcodes** controls if raw opcodes will be printed.
If the prog has line_info available, the source line will
be displayed by default. If **linum** is specified,
the filename, line number and line column will also be
displayed on top of the source line.
**bpftool prog pin** *PROG* *FILE* **bpftool prog pin** *PROG* *FILE*
Pin program *PROG* as *FILE*. Pin program *PROG* as *FILE*.
......
...@@ -191,7 +191,7 @@ _bpftool() ...@@ -191,7 +191,7 @@ _bpftool()
# Deal with simplest keywords # Deal with simplest keywords
case $prev in case $prev in
help|hex|opcodes|visual) help|hex|opcodes|visual|linum)
return 0 return 0
;; ;;
tag) tag)
...@@ -278,10 +278,10 @@ _bpftool() ...@@ -278,10 +278,10 @@ _bpftool()
*) *)
_bpftool_once_attr 'file' _bpftool_once_attr 'file'
if _bpftool_search_list 'xlated'; then if _bpftool_search_list 'xlated'; then
COMPREPLY+=( $( compgen -W 'opcodes visual' -- \ COMPREPLY+=( $( compgen -W 'opcodes visual linum' -- \
"$cur" ) ) "$cur" ) )
else else
COMPREPLY+=( $( compgen -W 'opcodes' -- \ COMPREPLY+=( $( compgen -W 'opcodes linum' -- \
"$cur" ) ) "$cur" ) )
fi fi
return 0 return 0
......
...@@ -385,3 +385,67 @@ void btf_dumper_type_only(const struct btf *btf, __u32 type_id, char *func_sig, ...@@ -385,3 +385,67 @@ void btf_dumper_type_only(const struct btf *btf, __u32 type_id, char *func_sig,
if (err < 0) if (err < 0)
func_sig[0] = '\0'; func_sig[0] = '\0';
} }
static const char *ltrim(const char *s)
{
while (isspace(*s))
s++;
return s;
}
void btf_dump_linfo_plain(const struct btf *btf,
const struct bpf_line_info *linfo,
const char *prefix, bool linum)
{
const char *line = btf__name_by_offset(btf, linfo->line_off);
if (!line)
return;
line = ltrim(line);
if (!prefix)
prefix = "";
if (linum) {
const char *file = btf__name_by_offset(btf, linfo->file_name_off);
/* More forgiving on file because linum option is
* expected to provide more info than the already
* available src line.
*/
if (!file)
file = "";
printf("%s%s [file:%s line_num:%u line_col:%u]\n",
prefix, line, file,
BPF_LINE_INFO_LINE_NUM(linfo->line_col),
BPF_LINE_INFO_LINE_COL(linfo->line_col));
} else {
printf("%s%s\n", prefix, line);
}
}
void btf_dump_linfo_json(const struct btf *btf,
const struct bpf_line_info *linfo, bool linum)
{
const char *line = btf__name_by_offset(btf, linfo->line_off);
if (line)
jsonw_string_field(json_wtr, "src", ltrim(line));
if (linum) {
const char *file = btf__name_by_offset(btf, linfo->file_name_off);
if (file)
jsonw_string_field(json_wtr, "file", file);
if (BPF_LINE_INFO_LINE_NUM(linfo->line_col))
jsonw_int_field(json_wtr, "line_num",
BPF_LINE_INFO_LINE_NUM(linfo->line_col));
if (BPF_LINE_INFO_LINE_COL(linfo->line_col))
jsonw_int_field(json_wtr, "line_col",
BPF_LINE_INFO_LINE_COL(linfo->line_col));
}
}
...@@ -21,6 +21,7 @@ ...@@ -21,6 +21,7 @@
#include <dis-asm.h> #include <dis-asm.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <limits.h> #include <limits.h>
#include <libbpf.h>
#include "json_writer.h" #include "json_writer.h"
#include "main.h" #include "main.h"
...@@ -68,10 +69,16 @@ static int fprintf_json(void *out, const char *fmt, ...) ...@@ -68,10 +69,16 @@ static int fprintf_json(void *out, const char *fmt, ...)
} }
void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
const char *arch, const char *disassembler_options) const char *arch, const char *disassembler_options,
const struct btf *btf,
const struct bpf_prog_linfo *prog_linfo,
__u64 func_ksym, unsigned int func_idx,
bool linum)
{ {
const struct bpf_line_info *linfo = NULL;
disassembler_ftype disassemble; disassembler_ftype disassemble;
struct disassemble_info info; struct disassemble_info info;
unsigned int nr_skip = 0;
int count, i, pc = 0; int count, i, pc = 0;
char tpath[PATH_MAX]; char tpath[PATH_MAX];
bfd *bfdf; bfd *bfdf;
...@@ -127,12 +134,26 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, ...@@ -127,12 +134,26 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
if (json_output) if (json_output)
jsonw_start_array(json_wtr); jsonw_start_array(json_wtr);
do { do {
if (prog_linfo) {
linfo = bpf_prog_linfo__lfind_addr_func(prog_linfo,
func_ksym + pc,
func_idx,
nr_skip);
if (linfo)
nr_skip++;
}
if (json_output) { if (json_output) {
jsonw_start_object(json_wtr); jsonw_start_object(json_wtr);
oper_count = 0; oper_count = 0;
if (linfo)
btf_dump_linfo_json(btf, linfo, linum);
jsonw_name(json_wtr, "pc"); jsonw_name(json_wtr, "pc");
jsonw_printf(json_wtr, "\"0x%x\"", pc); jsonw_printf(json_wtr, "\"0x%x\"", pc);
} else { } else {
if (linfo)
btf_dump_linfo_plain(btf, linfo, "; ",
linum);
printf("%4x:\t", pc); printf("%4x:\t", pc);
} }
......
...@@ -138,6 +138,9 @@ struct pinned_obj { ...@@ -138,6 +138,9 @@ struct pinned_obj {
struct hlist_node hash; struct hlist_node hash;
}; };
struct btf;
struct bpf_line_info;
int build_pinned_obj_table(struct pinned_obj_table *table, int build_pinned_obj_table(struct pinned_obj_table *table,
enum bpf_obj_type type); enum bpf_obj_type type);
void delete_pinned_obj_table(struct pinned_obj_table *tab); void delete_pinned_obj_table(struct pinned_obj_table *tab);
...@@ -175,13 +178,23 @@ int map_parse_fd(int *argc, char ***argv); ...@@ -175,13 +178,23 @@ int map_parse_fd(int *argc, char ***argv);
int map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len); int map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len);
#ifdef HAVE_LIBBFD_SUPPORT #ifdef HAVE_LIBBFD_SUPPORT
struct bpf_prog_linfo;
void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
const char *arch, const char *disassembler_options); const char *arch, const char *disassembler_options,
const struct btf *btf,
const struct bpf_prog_linfo *prog_linfo,
__u64 func_ksym, unsigned int func_idx,
bool linum);
int disasm_init(void); int disasm_init(void);
#else #else
static inline static inline
void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
const char *arch, const char *disassembler_options) const char *arch, const char *disassembler_options,
const struct btf *btf,
const struct bpf_prog_linfo *prog_linfo,
__u64 func_ksym, unsigned int func_idx,
bool linum)
{ {
} }
static inline int disasm_init(void) static inline int disasm_init(void)
...@@ -217,6 +230,12 @@ int btf_dumper_type(const struct btf_dumper *d, __u32 type_id, ...@@ -217,6 +230,12 @@ int btf_dumper_type(const struct btf_dumper *d, __u32 type_id,
void btf_dumper_type_only(const struct btf *btf, __u32 func_type_id, void btf_dumper_type_only(const struct btf *btf, __u32 func_type_id,
char *func_only, int size); char *func_only, int size);
void btf_dump_linfo_plain(const struct btf *btf,
const struct bpf_line_info *linfo,
const char *prefix, bool linum);
void btf_dump_linfo_json(const struct btf *btf,
const struct bpf_line_info *linfo, bool linum);
struct nlattr; struct nlattr;
struct ifinfomsg; struct ifinfomsg;
struct tcmsg; struct tcmsg;
......
...@@ -423,24 +423,26 @@ static int do_show(int argc, char **argv) ...@@ -423,24 +423,26 @@ static int do_show(int argc, char **argv)
static int do_dump(int argc, char **argv) static int do_dump(int argc, char **argv)
{ {
unsigned int finfo_rec_size, linfo_rec_size, jited_linfo_rec_size;
void *func_info = NULL, *linfo = NULL, *jited_linfo = NULL;
unsigned int finfo_cnt, linfo_cnt = 0, jited_linfo_cnt = 0;
struct bpf_prog_linfo *prog_linfo = NULL;
unsigned long *func_ksyms = NULL; unsigned long *func_ksyms = NULL;
struct bpf_prog_info info = {}; struct bpf_prog_info info = {};
unsigned int *func_lens = NULL; unsigned int *func_lens = NULL;
const char *disasm_opt = NULL; const char *disasm_opt = NULL;
unsigned int finfo_rec_size;
unsigned int nr_func_ksyms; unsigned int nr_func_ksyms;
unsigned int nr_func_lens; unsigned int nr_func_lens;
struct dump_data dd = {}; struct dump_data dd = {};
__u32 len = sizeof(info); __u32 len = sizeof(info);
struct btf *btf = NULL; struct btf *btf = NULL;
void *func_info = NULL;
unsigned int finfo_cnt;
unsigned int buf_size; unsigned int buf_size;
char *filepath = NULL; char *filepath = NULL;
bool opcodes = false; bool opcodes = false;
bool visual = false; bool visual = false;
char func_sig[1024]; char func_sig[1024];
unsigned char *buf; unsigned char *buf;
bool linum = false;
__u32 *member_len; __u32 *member_len;
__u64 *member_ptr; __u64 *member_ptr;
ssize_t n; ssize_t n;
...@@ -484,6 +486,9 @@ static int do_dump(int argc, char **argv) ...@@ -484,6 +486,9 @@ static int do_dump(int argc, char **argv)
} else if (is_prefix(*argv, "visual")) { } else if (is_prefix(*argv, "visual")) {
visual = true; visual = true;
NEXT_ARG(); NEXT_ARG();
} else if (is_prefix(*argv, "linum")) {
linum = true;
NEXT_ARG();
} }
if (argc) { if (argc) {
...@@ -543,6 +548,32 @@ static int do_dump(int argc, char **argv) ...@@ -543,6 +548,32 @@ static int do_dump(int argc, char **argv)
} }
} }
linfo_rec_size = info.line_info_rec_size;
if (info.line_info_cnt && linfo_rec_size && info.btf_id) {
linfo_cnt = info.line_info_cnt;
linfo = malloc(linfo_cnt * linfo_rec_size);
if (!linfo) {
p_err("mem alloc failed");
close(fd);
goto err_free;
}
}
jited_linfo_rec_size = info.jited_line_info_rec_size;
if (info.jited_line_info_cnt &&
jited_linfo_rec_size &&
info.nr_jited_ksyms &&
info.nr_jited_func_lens &&
info.btf_id) {
jited_linfo_cnt = info.jited_line_info_cnt;
jited_linfo = malloc(jited_linfo_cnt * jited_linfo_rec_size);
if (!jited_linfo) {
p_err("mem alloc failed");
close(fd);
goto err_free;
}
}
memset(&info, 0, sizeof(info)); memset(&info, 0, sizeof(info));
*member_ptr = ptr_to_u64(buf); *member_ptr = ptr_to_u64(buf);
...@@ -554,6 +585,13 @@ static int do_dump(int argc, char **argv) ...@@ -554,6 +585,13 @@ static int do_dump(int argc, char **argv)
info.func_info_cnt = finfo_cnt; info.func_info_cnt = finfo_cnt;
info.func_info_rec_size = finfo_rec_size; info.func_info_rec_size = finfo_rec_size;
info.func_info = ptr_to_u64(func_info); info.func_info = ptr_to_u64(func_info);
info.line_info_cnt = linfo_cnt;
info.line_info_rec_size = linfo_rec_size;
info.line_info = ptr_to_u64(linfo);
info.jited_line_info_cnt = jited_linfo_cnt;
info.jited_line_info_rec_size = jited_linfo_rec_size;
info.jited_line_info = ptr_to_u64(jited_linfo);
err = bpf_obj_get_info_by_fd(fd, &info, &len); err = bpf_obj_get_info_by_fd(fd, &info, &len);
close(fd); close(fd);
...@@ -596,6 +634,30 @@ static int do_dump(int argc, char **argv) ...@@ -596,6 +634,30 @@ static int do_dump(int argc, char **argv)
finfo_cnt = 0; finfo_cnt = 0;
} }
if (linfo && info.line_info_cnt != linfo_cnt) {
p_err("incorrect line_info_cnt %u vs. expected %u",
info.line_info_cnt, linfo_cnt);
goto err_free;
}
if (info.line_info_rec_size != linfo_rec_size) {
p_err("incorrect line_info_rec_size %u vs. expected %u",
info.line_info_rec_size, linfo_rec_size);
goto err_free;
}
if (jited_linfo && info.jited_line_info_cnt != jited_linfo_cnt) {
p_err("incorrect jited_line_info_cnt %u vs. expected %u",
info.jited_line_info_cnt, jited_linfo_cnt);
goto err_free;
}
if (info.jited_line_info_rec_size != jited_linfo_rec_size) {
p_err("incorrect jited_line_info_rec_size %u vs. expected %u",
info.jited_line_info_rec_size, jited_linfo_rec_size);
goto err_free;
}
if ((member_len == &info.jited_prog_len && if ((member_len == &info.jited_prog_len &&
info.jited_prog_insns == 0) || info.jited_prog_insns == 0) ||
(member_len == &info.xlated_prog_len && (member_len == &info.xlated_prog_len &&
...@@ -609,6 +671,12 @@ static int do_dump(int argc, char **argv) ...@@ -609,6 +671,12 @@ static int do_dump(int argc, char **argv)
goto err_free; goto err_free;
} }
if (linfo_cnt) {
prog_linfo = bpf_prog_linfo__new(&info);
if (!prog_linfo)
p_err("error in processing bpf_line_info. continue without it.");
}
if (filepath) { if (filepath) {
fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600); fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (fd < 0) { if (fd < 0) {
...@@ -690,8 +758,11 @@ static int do_dump(int argc, char **argv) ...@@ -690,8 +758,11 @@ static int do_dump(int argc, char **argv)
printf("%s:\n", sym_name); printf("%s:\n", sym_name);
} }
disasm_print_insn(img, lens[i], opcodes, name, disasm_print_insn(img, lens[i], opcodes,
disasm_opt); name, disasm_opt, btf,
prog_linfo, ksyms[i], i,
linum);
img += lens[i]; img += lens[i];
if (json_output) if (json_output)
...@@ -704,7 +775,7 @@ static int do_dump(int argc, char **argv) ...@@ -704,7 +775,7 @@ static int do_dump(int argc, char **argv)
jsonw_end_array(json_wtr); jsonw_end_array(json_wtr);
} else { } else {
disasm_print_insn(buf, *member_len, opcodes, name, disasm_print_insn(buf, *member_len, opcodes, name,
disasm_opt); disasm_opt, btf, NULL, 0, 0, false);
} }
} else if (visual) { } else if (visual) {
if (json_output) if (json_output)
...@@ -718,11 +789,14 @@ static int do_dump(int argc, char **argv) ...@@ -718,11 +789,14 @@ static int do_dump(int argc, char **argv)
dd.btf = btf; dd.btf = btf;
dd.func_info = func_info; dd.func_info = func_info;
dd.finfo_rec_size = finfo_rec_size; dd.finfo_rec_size = finfo_rec_size;
dd.prog_linfo = prog_linfo;
if (json_output) if (json_output)
dump_xlated_json(&dd, buf, *member_len, opcodes); dump_xlated_json(&dd, buf, *member_len, opcodes,
linum);
else else
dump_xlated_plain(&dd, buf, *member_len, opcodes); dump_xlated_plain(&dd, buf, *member_len, opcodes,
linum);
kernel_syms_destroy(&dd); kernel_syms_destroy(&dd);
} }
...@@ -730,6 +804,9 @@ static int do_dump(int argc, char **argv) ...@@ -730,6 +804,9 @@ static int do_dump(int argc, char **argv)
free(func_ksyms); free(func_ksyms);
free(func_lens); free(func_lens);
free(func_info); free(func_info);
free(linfo);
free(jited_linfo);
bpf_prog_linfo__free(prog_linfo);
return 0; return 0;
err_free: err_free:
...@@ -737,6 +814,9 @@ static int do_dump(int argc, char **argv) ...@@ -737,6 +814,9 @@ static int do_dump(int argc, char **argv)
free(func_ksyms); free(func_ksyms);
free(func_lens); free(func_lens);
free(func_info); free(func_info);
free(linfo);
free(jited_linfo);
bpf_prog_linfo__free(prog_linfo);
return -1; return -1;
} }
...@@ -1138,8 +1218,8 @@ static int do_help(int argc, char **argv) ...@@ -1138,8 +1218,8 @@ static int do_help(int argc, char **argv)
fprintf(stderr, fprintf(stderr,
"Usage: %s %s { show | list } [PROG]\n" "Usage: %s %s { show | list } [PROG]\n"
" %s %s dump xlated PROG [{ file FILE | opcodes | visual }]\n" " %s %s dump xlated PROG [{ file FILE | opcodes | visual | linum }]\n"
" %s %s dump jited PROG [{ file FILE | opcodes }]\n" " %s %s dump jited PROG [{ file FILE | opcodes | linum }]\n"
" %s %s pin PROG FILE\n" " %s %s pin PROG FILE\n"
" %s %s { load | loadall } OBJ PATH \\\n" " %s %s { load | loadall } OBJ PATH \\\n"
" [type TYPE] [dev NAME] \\\n" " [type TYPE] [dev NAME] \\\n"
......
...@@ -41,6 +41,7 @@ ...@@ -41,6 +41,7 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <sys/types.h> #include <sys/types.h>
#include <libbpf.h>
#include "disasm.h" #include "disasm.h"
#include "json_writer.h" #include "json_writer.h"
...@@ -234,8 +235,9 @@ static const char *print_imm(void *private_data, ...@@ -234,8 +235,9 @@ static const char *print_imm(void *private_data,
} }
void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
bool opcodes) bool opcodes, bool linum)
{ {
const struct bpf_prog_linfo *prog_linfo = dd->prog_linfo;
const struct bpf_insn_cbs cbs = { const struct bpf_insn_cbs cbs = {
.cb_print = print_insn_json, .cb_print = print_insn_json,
.cb_call = print_call, .cb_call = print_call,
...@@ -246,6 +248,7 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, ...@@ -246,6 +248,7 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
struct bpf_insn *insn = buf; struct bpf_insn *insn = buf;
struct btf *btf = dd->btf; struct btf *btf = dd->btf;
bool double_insn = false; bool double_insn = false;
unsigned int nr_skip = 0;
char func_sig[1024]; char func_sig[1024];
unsigned int i; unsigned int i;
...@@ -273,6 +276,16 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, ...@@ -273,6 +276,16 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
} }
} }
if (prog_linfo) {
const struct bpf_line_info *linfo;
linfo = bpf_prog_linfo__lfind(prog_linfo, i, nr_skip);
if (linfo) {
btf_dump_linfo_json(btf, linfo, linum);
nr_skip++;
}
}
jsonw_name(json_wtr, "disasm"); jsonw_name(json_wtr, "disasm");
print_bpf_insn(&cbs, insn + i, true); print_bpf_insn(&cbs, insn + i, true);
...@@ -307,8 +320,9 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, ...@@ -307,8 +320,9 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
} }
void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
bool opcodes) bool opcodes, bool linum)
{ {
const struct bpf_prog_linfo *prog_linfo = dd->prog_linfo;
const struct bpf_insn_cbs cbs = { const struct bpf_insn_cbs cbs = {
.cb_print = print_insn, .cb_print = print_insn,
.cb_call = print_call, .cb_call = print_call,
...@@ -318,6 +332,7 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, ...@@ -318,6 +332,7 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
struct bpf_func_info *record; struct bpf_func_info *record;
struct bpf_insn *insn = buf; struct bpf_insn *insn = buf;
struct btf *btf = dd->btf; struct btf *btf = dd->btf;
unsigned int nr_skip = 0;
bool double_insn = false; bool double_insn = false;
char func_sig[1024]; char func_sig[1024];
unsigned int i; unsigned int i;
...@@ -340,6 +355,17 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, ...@@ -340,6 +355,17 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
} }
} }
if (prog_linfo) {
const struct bpf_line_info *linfo;
linfo = bpf_prog_linfo__lfind(prog_linfo, i, nr_skip);
if (linfo) {
btf_dump_linfo_plain(btf, linfo, "; ",
linum);
nr_skip++;
}
}
double_insn = insn[i].code == (BPF_LD | BPF_IMM | BPF_DW); double_insn = insn[i].code == (BPF_LD | BPF_IMM | BPF_DW);
printf("% 4d: ", i); printf("% 4d: ", i);
......
...@@ -40,6 +40,8 @@ ...@@ -40,6 +40,8 @@
#define SYM_MAX_NAME 256 #define SYM_MAX_NAME 256
struct bpf_prog_linfo;
struct kernel_sym { struct kernel_sym {
unsigned long address; unsigned long address;
char name[SYM_MAX_NAME]; char name[SYM_MAX_NAME];
...@@ -54,6 +56,7 @@ struct dump_data { ...@@ -54,6 +56,7 @@ struct dump_data {
struct btf *btf; struct btf *btf;
void *func_info; void *func_info;
__u32 finfo_rec_size; __u32 finfo_rec_size;
const struct bpf_prog_linfo *prog_linfo;
char scratch_buff[SYM_MAX_NAME + 8]; char scratch_buff[SYM_MAX_NAME + 8];
}; };
...@@ -61,9 +64,9 @@ void kernel_syms_load(struct dump_data *dd); ...@@ -61,9 +64,9 @@ void kernel_syms_load(struct dump_data *dd);
void kernel_syms_destroy(struct dump_data *dd); void kernel_syms_destroy(struct dump_data *dd);
struct kernel_sym *kernel_syms_search(struct dump_data *dd, unsigned long key); struct kernel_sym *kernel_syms_search(struct dump_data *dd, unsigned long key);
void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
bool opcodes); bool opcodes, bool linum);
void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
bool opcodes); bool opcodes, bool linum);
void dump_xlated_for_graph(struct dump_data *dd, void *buf, void *buf_end, void dump_xlated_for_graph(struct dump_data *dd, void *buf, void *buf_end,
unsigned int start_index); unsigned int start_index);
......
...@@ -356,6 +356,9 @@ union bpf_attr { ...@@ -356,6 +356,9 @@ union bpf_attr {
__u32 func_info_rec_size; /* userspace bpf_func_info size */ __u32 func_info_rec_size; /* userspace bpf_func_info size */
__aligned_u64 func_info; /* func info */ __aligned_u64 func_info; /* func info */
__u32 func_info_cnt; /* number of bpf_func_info records */ __u32 func_info_cnt; /* number of bpf_func_info records */
__u32 line_info_rec_size; /* userspace bpf_line_info size */
__aligned_u64 line_info; /* line info */
__u32 line_info_cnt; /* number of bpf_line_info records */
}; };
struct { /* anonymous struct used by BPF_OBJ_* commands */ struct { /* anonymous struct used by BPF_OBJ_* commands */
...@@ -2679,6 +2682,12 @@ struct bpf_prog_info { ...@@ -2679,6 +2682,12 @@ struct bpf_prog_info {
__u32 func_info_rec_size; __u32 func_info_rec_size;
__aligned_u64 func_info; __aligned_u64 func_info;
__u32 func_info_cnt; __u32 func_info_cnt;
__u32 line_info_cnt;
__aligned_u64 line_info;
__aligned_u64 jited_line_info;
__u32 jited_line_info_cnt;
__u32 line_info_rec_size;
__u32 jited_line_info_rec_size;
} __attribute__((aligned(8))); } __attribute__((aligned(8)));
struct bpf_map_info { struct bpf_map_info {
...@@ -2995,4 +3004,14 @@ struct bpf_func_info { ...@@ -2995,4 +3004,14 @@ struct bpf_func_info {
__u32 type_id; __u32 type_id;
}; };
#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10)
#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff)
struct bpf_line_info {
__u32 insn_off;
__u32 file_name_off;
__u32 line_off;
__u32 line_col;
};
#endif /* _UAPI__LINUX_BPF_H__ */ #endif /* _UAPI__LINUX_BPF_H__ */
libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o netlink.o libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o netlink.o bpf_prog_linfo.o
...@@ -173,11 +173,36 @@ int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name, ...@@ -173,11 +173,36 @@ int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name,
-1); -1);
} }
static void *
alloc_zero_tailing_info(const void *orecord, __u32 cnt,
__u32 actual_rec_size, __u32 expected_rec_size)
{
__u64 info_len = actual_rec_size * cnt;
void *info, *nrecord;
int i;
info = malloc(info_len);
if (!info)
return NULL;
/* zero out bytes kernel does not understand */
nrecord = info;
for (i = 0; i < cnt; i++) {
memcpy(nrecord, orecord, expected_rec_size);
memset(nrecord + expected_rec_size, 0,
actual_rec_size - expected_rec_size);
orecord += actual_rec_size;
nrecord += actual_rec_size;
}
return info;
}
int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
char *log_buf, size_t log_buf_sz) char *log_buf, size_t log_buf_sz)
{ {
void *finfo = NULL, *linfo = NULL;
union bpf_attr attr; union bpf_attr attr;
void *finfo = NULL;
__u32 name_len; __u32 name_len;
int fd; int fd;
...@@ -201,53 +226,58 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, ...@@ -201,53 +226,58 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
attr.func_info_rec_size = load_attr->func_info_rec_size; attr.func_info_rec_size = load_attr->func_info_rec_size;
attr.func_info_cnt = load_attr->func_info_cnt; attr.func_info_cnt = load_attr->func_info_cnt;
attr.func_info = ptr_to_u64(load_attr->func_info); attr.func_info = ptr_to_u64(load_attr->func_info);
attr.line_info_rec_size = load_attr->line_info_rec_size;
attr.line_info_cnt = load_attr->line_info_cnt;
attr.line_info = ptr_to_u64(load_attr->line_info);
memcpy(attr.prog_name, load_attr->name, memcpy(attr.prog_name, load_attr->name,
min(name_len, BPF_OBJ_NAME_LEN - 1)); min(name_len, BPF_OBJ_NAME_LEN - 1));
fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
if (fd >= 0 || !log_buf || !log_buf_sz) if (fd >= 0)
return fd; return fd;
/* After bpf_prog_load, the kernel may modify certain attributes /* After bpf_prog_load, the kernel may modify certain attributes
* to give user space a hint how to deal with loading failure. * to give user space a hint how to deal with loading failure.
* Check to see whether we can make some changes and load again. * Check to see whether we can make some changes and load again.
*/ */
if (errno == E2BIG && attr.func_info_cnt && while (errno == E2BIG && (!finfo || !linfo)) {
attr.func_info_rec_size < load_attr->func_info_rec_size) { if (!finfo && attr.func_info_cnt &&
__u32 actual_rec_size = load_attr->func_info_rec_size; attr.func_info_rec_size < load_attr->func_info_rec_size) {
__u32 expected_rec_size = attr.func_info_rec_size; /* try with corrected func info records */
__u32 finfo_cnt = load_attr->func_info_cnt; finfo = alloc_zero_tailing_info(load_attr->func_info,
__u64 finfo_len = actual_rec_size * finfo_cnt; load_attr->func_info_cnt,
const void *orecord; load_attr->func_info_rec_size,
void *nrecord; attr.func_info_rec_size);
int i; if (!finfo)
goto done;
finfo = malloc(finfo_len);
if (!finfo) attr.func_info = ptr_to_u64(finfo);
/* further try with log buffer won't help */ attr.func_info_rec_size = load_attr->func_info_rec_size;
return fd; } else if (!linfo && attr.line_info_cnt &&
attr.line_info_rec_size <
/* zero out bytes kernel does not understand */ load_attr->line_info_rec_size) {
orecord = load_attr->func_info; linfo = alloc_zero_tailing_info(load_attr->line_info,
nrecord = finfo; load_attr->line_info_cnt,
for (i = 0; i < load_attr->func_info_cnt; i++) { load_attr->line_info_rec_size,
memcpy(nrecord, orecord, expected_rec_size); attr.line_info_rec_size);
memset(nrecord + expected_rec_size, 0, if (!linfo)
actual_rec_size - expected_rec_size); goto done;
orecord += actual_rec_size;
nrecord += actual_rec_size; attr.line_info = ptr_to_u64(linfo);
attr.line_info_rec_size = load_attr->line_info_rec_size;
} else {
break;
} }
/* try with corrected func info records */
attr.func_info = ptr_to_u64(finfo);
attr.func_info_rec_size = load_attr->func_info_rec_size;
fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
if (fd >= 0 || !log_buf || !log_buf_sz) if (fd >= 0)
goto done; goto done;
} }
if (!log_buf || !log_buf_sz)
goto done;
/* Try again with log */ /* Try again with log */
attr.log_buf = ptr_to_u64(log_buf); attr.log_buf = ptr_to_u64(log_buf);
attr.log_size = log_buf_sz; attr.log_size = log_buf_sz;
...@@ -256,6 +286,7 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, ...@@ -256,6 +286,7 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
done: done:
free(finfo); free(finfo);
free(linfo);
return fd; return fd;
} }
......
...@@ -82,6 +82,9 @@ struct bpf_load_program_attr { ...@@ -82,6 +82,9 @@ struct bpf_load_program_attr {
__u32 func_info_rec_size; __u32 func_info_rec_size;
const void *func_info; const void *func_info;
__u32 func_info_cnt; __u32 func_info_cnt;
__u32 line_info_rec_size;
const void *line_info;
__u32 line_info_cnt;
}; };
/* Flags to direct loading requirements */ /* Flags to direct loading requirements */
......
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
/* Copyright (c) 2018 Facebook */
#include <string.h>
#include <stdlib.h>
#include <linux/err.h>
#include <linux/bpf.h>
#include "libbpf.h"
#ifndef min
#define min(x, y) ((x) < (y) ? (x) : (y))
#endif
struct bpf_prog_linfo {
void *raw_linfo;
void *raw_jited_linfo;
__u32 *nr_jited_linfo_per_func;
__u32 *jited_linfo_func_idx;
__u32 nr_linfo;
__u32 nr_jited_func;
__u32 rec_size;
__u32 jited_rec_size;
};
static int dissect_jited_func(struct bpf_prog_linfo *prog_linfo,
const __u64 *ksym_func, const __u32 *ksym_len)
{
__u32 nr_jited_func, nr_linfo;
const void *raw_jited_linfo;
const __u64 *jited_linfo;
__u64 last_jited_linfo;
/*
* Index to raw_jited_linfo:
* i: Index for searching the next ksym_func
* prev_i: Index to the last found ksym_func
*/
__u32 i, prev_i;
__u32 f; /* Index to ksym_func */
raw_jited_linfo = prog_linfo->raw_jited_linfo;
jited_linfo = raw_jited_linfo;
if (ksym_func[0] != *jited_linfo)
goto errout;
prog_linfo->jited_linfo_func_idx[0] = 0;
nr_jited_func = prog_linfo->nr_jited_func;
nr_linfo = prog_linfo->nr_linfo;
for (prev_i = 0, i = 1, f = 1;
i < nr_linfo && f < nr_jited_func;
i++) {
raw_jited_linfo += prog_linfo->jited_rec_size;
last_jited_linfo = *jited_linfo;
jited_linfo = raw_jited_linfo;
if (ksym_func[f] == *jited_linfo) {
prog_linfo->jited_linfo_func_idx[f] = i;
/* Sanity check */
if (last_jited_linfo - ksym_func[f - 1] + 1 >
ksym_len[f - 1])
goto errout;
prog_linfo->nr_jited_linfo_per_func[f - 1] =
i - prev_i;
prev_i = i;
/*
* The ksym_func[f] is found in jited_linfo.
* Look for the next one.
*/
f++;
} else if (*jited_linfo <= last_jited_linfo) {
/* Ensure the addr is increasing _within_ a func */
goto errout;
}
}
if (f != nr_jited_func)
goto errout;
prog_linfo->nr_jited_linfo_per_func[nr_jited_func - 1] =
nr_linfo - prev_i;
return 0;
errout:
return -EINVAL;
}
void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo)
{
if (!prog_linfo)
return;
free(prog_linfo->raw_linfo);
free(prog_linfo->raw_jited_linfo);
free(prog_linfo->nr_jited_linfo_per_func);
free(prog_linfo->jited_linfo_func_idx);
free(prog_linfo);
}
struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info)
{
struct bpf_prog_linfo *prog_linfo;
__u32 nr_linfo, nr_jited_func;
nr_linfo = info->line_info_cnt;
/*
* Test !info->line_info because the kernel may NULL
* the ptr if kernel.kptr_restrict is set.
*/
if (!nr_linfo || !info->line_info)
return NULL;
/*
* The min size that bpf_prog_linfo has to access for
* searching purpose.
*/
if (info->line_info_rec_size <
offsetof(struct bpf_line_info, file_name_off))
return NULL;
prog_linfo = calloc(1, sizeof(*prog_linfo));
if (!prog_linfo)
return NULL;
/* Copy xlated line_info */
prog_linfo->nr_linfo = nr_linfo;
prog_linfo->rec_size = info->line_info_rec_size;
prog_linfo->raw_linfo = malloc(nr_linfo * prog_linfo->rec_size);
if (!prog_linfo->raw_linfo)
goto err_free;
memcpy(prog_linfo->raw_linfo, (void *)(long)info->line_info,
nr_linfo * prog_linfo->rec_size);
nr_jited_func = info->nr_jited_ksyms;
if (!nr_jited_func ||
!info->jited_line_info ||
info->jited_line_info_cnt != nr_linfo ||
info->jited_line_info_rec_size < sizeof(__u64) ||
info->nr_jited_func_lens != nr_jited_func ||
!info->jited_ksyms ||
!info->jited_func_lens)
/* Not enough info to provide jited_line_info */
return prog_linfo;
/* Copy jited_line_info */
prog_linfo->nr_jited_func = nr_jited_func;
prog_linfo->jited_rec_size = info->jited_line_info_rec_size;
prog_linfo->raw_jited_linfo = malloc(nr_linfo *
prog_linfo->jited_rec_size);
if (!prog_linfo->raw_jited_linfo)
goto err_free;
memcpy(prog_linfo->raw_jited_linfo,
(void *)(long)info->jited_line_info,
nr_linfo * prog_linfo->jited_rec_size);
/* Number of jited_line_info per jited func */
prog_linfo->nr_jited_linfo_per_func = malloc(nr_jited_func *
sizeof(__u32));
if (!prog_linfo->nr_jited_linfo_per_func)
goto err_free;
/*
* For each jited func,
* the start idx to the "linfo" and "jited_linfo" array,
*/
prog_linfo->jited_linfo_func_idx = malloc(nr_jited_func *
sizeof(__u32));
if (!prog_linfo->jited_linfo_func_idx)
goto err_free;
if (dissect_jited_func(prog_linfo,
(__u64 *)(long)info->jited_ksyms,
(__u32 *)(long)info->jited_func_lens))
goto err_free;
return prog_linfo;
err_free:
bpf_prog_linfo__free(prog_linfo);
return NULL;
}
const struct bpf_line_info *
bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo,
__u64 addr, __u32 func_idx, __u32 nr_skip)
{
__u32 jited_rec_size, rec_size, nr_linfo, start, i;
const void *raw_jited_linfo, *raw_linfo;
const __u64 *jited_linfo;
if (func_idx >= prog_linfo->nr_jited_func)
return NULL;
nr_linfo = prog_linfo->nr_jited_linfo_per_func[func_idx];
if (nr_skip >= nr_linfo)
return NULL;
start = prog_linfo->jited_linfo_func_idx[func_idx] + nr_skip;
jited_rec_size = prog_linfo->jited_rec_size;
raw_jited_linfo = prog_linfo->raw_jited_linfo +
(start * jited_rec_size);
jited_linfo = raw_jited_linfo;
if (addr < *jited_linfo)
return NULL;
nr_linfo -= nr_skip;
rec_size = prog_linfo->rec_size;
raw_linfo = prog_linfo->raw_linfo + (start * rec_size);
for (i = 0; i < nr_linfo; i++) {
if (addr < *jited_linfo)
break;
raw_linfo += rec_size;
raw_jited_linfo += jited_rec_size;
jited_linfo = raw_jited_linfo;
}
return raw_linfo - rec_size;
}
const struct bpf_line_info *
bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo,
__u32 insn_off, __u32 nr_skip)
{
const struct bpf_line_info *linfo;
__u32 rec_size, nr_linfo, i;
const void *raw_linfo;
nr_linfo = prog_linfo->nr_linfo;
if (nr_skip >= nr_linfo)
return NULL;
rec_size = prog_linfo->rec_size;
raw_linfo = prog_linfo->raw_linfo + (nr_skip * rec_size);
linfo = raw_linfo;
if (insn_off < linfo->insn_off)
return NULL;
nr_linfo -= nr_skip;
for (i = 0; i < nr_linfo; i++) {
if (insn_off < linfo->insn_off)
break;
raw_linfo += rec_size;
linfo = raw_linfo;
}
return raw_linfo - rec_size;
}
...@@ -37,10 +37,27 @@ struct btf { ...@@ -37,10 +37,27 @@ struct btf {
int fd; int fd;
}; };
struct btf_ext_info {
/*
* info points to a deep copy of the individual info section
* (e.g. func_info and line_info) from the .BTF.ext.
* It does not include the __u32 rec_size.
*/
void *info;
__u32 rec_size;
__u32 len;
};
struct btf_ext { struct btf_ext {
void *func_info; struct btf_ext_info func_info;
__u32 func_info_rec_size; struct btf_ext_info line_info;
__u32 func_info_len; };
struct btf_ext_info_sec {
__u32 sec_name_off;
__u32 num_info;
/* Followed by num_info * record_size number of bytes */
__u8 data[0];
}; };
/* The minimum bpf_func_info checked by the loader */ /* The minimum bpf_func_info checked by the loader */
...@@ -49,6 +66,14 @@ struct bpf_func_info_min { ...@@ -49,6 +66,14 @@ struct bpf_func_info_min {
__u32 type_id; __u32 type_id;
}; };
/* The minimum bpf_line_info checked by the loader */
struct bpf_line_info_min {
__u32 insn_off;
__u32 file_name_off;
__u32 line_off;
__u32 line_col;
};
static inline __u64 ptr_to_u64(const void *ptr) static inline __u64 ptr_to_u64(const void *ptr)
{ {
return (__u64) (unsigned long) ptr; return (__u64) (unsigned long) ptr;
...@@ -479,71 +504,147 @@ int btf__get_from_id(__u32 id, struct btf **btf) ...@@ -479,71 +504,147 @@ int btf__get_from_id(__u32 id, struct btf **btf)
return err; return err;
} }
static int btf_ext_validate_func_info(const void *finfo, __u32 size, struct btf_ext_sec_copy_param {
btf_print_fn_t err_log) __u32 off;
__u32 len;
__u32 min_rec_size;
struct btf_ext_info *ext_info;
const char *desc;
};
static int btf_ext_copy_info(struct btf_ext *btf_ext,
__u8 *data, __u32 data_size,
struct btf_ext_sec_copy_param *ext_sec,
btf_print_fn_t err_log)
{ {
int sec_hdrlen = sizeof(struct btf_sec_func_info); const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
__u32 size_left, num_records, record_size; const struct btf_ext_info_sec *sinfo;
const struct btf_sec_func_info *sinfo; struct btf_ext_info *ext_info;
__u64 total_record_size; __u32 info_left, record_size;
/* The start of the info sec (including the __u32 record_size). */
/* At least a func_info record size */ const void *info;
if (size < sizeof(__u32)) {
elog("BTF.ext func_info record size not found"); /* data and data_size do not include btf_ext_header from now on */
data = data + hdr->hdr_len;
data_size -= hdr->hdr_len;
if (ext_sec->off & 0x03) {
elog(".BTF.ext %s section is not aligned to 4 bytes\n",
ext_sec->desc);
return -EINVAL; return -EINVAL;
} }
/* The record size needs to meet below minimum standard */ if (data_size < ext_sec->off ||
record_size = *(__u32 *)finfo; ext_sec->len > data_size - ext_sec->off) {
if (record_size < sizeof(struct bpf_func_info_min) || elog("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n",
record_size % sizeof(__u32)) { ext_sec->desc, ext_sec->off, ext_sec->len);
elog("BTF.ext func_info invalid record size");
return -EINVAL; return -EINVAL;
} }
sinfo = finfo + sizeof(__u32); info = data + ext_sec->off;
size_left = size - sizeof(__u32); info_left = ext_sec->len;
/* If no func_info records, return failure now so .BTF.ext /* At least a record size */
* won't be used. if (info_left < sizeof(__u32)) {
*/ elog(".BTF.ext %s record size not found\n", ext_sec->desc);
if (!size_left) { return -EINVAL;
elog("BTF.ext no func info records"); }
/* The record size needs to meet the minimum standard */
record_size = *(__u32 *)info;
if (record_size < ext_sec->min_rec_size ||
record_size & 0x03) {
elog("%s section in .BTF.ext has invalid record size %u\n",
ext_sec->desc, record_size);
return -EINVAL; return -EINVAL;
} }
while (size_left) { sinfo = info + sizeof(__u32);
if (size_left < sec_hdrlen) { info_left -= sizeof(__u32);
elog("BTF.ext func_info header not found");
/* If no records, return failure now so .BTF.ext won't be used. */
if (!info_left) {
elog("%s section in .BTF.ext has no records", ext_sec->desc);
return -EINVAL;
}
while (info_left) {
unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
__u64 total_record_size;
__u32 num_records;
if (info_left < sec_hdrlen) {
elog("%s section header is not found in .BTF.ext\n",
ext_sec->desc);
return -EINVAL; return -EINVAL;
} }
num_records = sinfo->num_func_info; num_records = sinfo->num_info;
if (num_records == 0) { if (num_records == 0) {
elog("incorrect BTF.ext num_func_info"); elog("%s section has incorrect num_records in .BTF.ext\n",
ext_sec->desc);
return -EINVAL; return -EINVAL;
} }
total_record_size = sec_hdrlen + total_record_size = sec_hdrlen +
(__u64)num_records * record_size; (__u64)num_records * record_size;
if (size_left < total_record_size) { if (info_left < total_record_size) {
elog("incorrect BTF.ext num_func_info"); elog("%s section has incorrect num_records in .BTF.ext\n",
ext_sec->desc);
return -EINVAL; return -EINVAL;
} }
size_left -= total_record_size; info_left -= total_record_size;
sinfo = (void *)sinfo + total_record_size; sinfo = (void *)sinfo + total_record_size;
} }
ext_info = ext_sec->ext_info;
ext_info->len = ext_sec->len - sizeof(__u32);
ext_info->rec_size = record_size;
ext_info->info = malloc(ext_info->len);
if (!ext_info->info)
return -ENOMEM;
memcpy(ext_info->info, info + sizeof(__u32), ext_info->len);
return 0; return 0;
} }
static int btf_ext_copy_func_info(struct btf_ext *btf_ext,
__u8 *data, __u32 data_size,
btf_print_fn_t err_log)
{
const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
struct btf_ext_sec_copy_param param = {
.off = hdr->func_info_off,
.len = hdr->func_info_len,
.min_rec_size = sizeof(struct bpf_func_info_min),
.ext_info = &btf_ext->func_info,
.desc = "func_info"
};
return btf_ext_copy_info(btf_ext, data, data_size, &param, err_log);
}
static int btf_ext_copy_line_info(struct btf_ext *btf_ext,
__u8 *data, __u32 data_size,
btf_print_fn_t err_log)
{
const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
struct btf_ext_sec_copy_param param = {
.off = hdr->line_info_off,
.len = hdr->line_info_len,
.min_rec_size = sizeof(struct bpf_line_info_min),
.ext_info = &btf_ext->line_info,
.desc = "line_info",
};
return btf_ext_copy_info(btf_ext, data, data_size, &param, err_log);
}
static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, static int btf_ext_parse_hdr(__u8 *data, __u32 data_size,
btf_print_fn_t err_log) btf_print_fn_t err_log)
{ {
const struct btf_ext_header *hdr = (struct btf_ext_header *)data; const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
__u32 meta_left, last_func_info_pos;
void *finfo;
if (data_size < offsetof(struct btf_ext_header, func_info_off) || if (data_size < offsetof(struct btf_ext_header, func_info_off) ||
data_size < hdr->hdr_len) { data_size < hdr->hdr_len) {
...@@ -566,34 +667,12 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, ...@@ -566,34 +667,12 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size,
return -ENOTSUP; return -ENOTSUP;
} }
meta_left = data_size - hdr->hdr_len; if (data_size == hdr->hdr_len) {
if (!meta_left) {
elog("BTF.ext has no data\n"); elog("BTF.ext has no data\n");
return -EINVAL; return -EINVAL;
} }
if (meta_left < hdr->func_info_off) { return 0;
elog("Invalid BTF.ext func_info section offset:%u\n",
hdr->func_info_off);
return -EINVAL;
}
if (hdr->func_info_off & 0x03) {
elog("BTF.ext func_info section is not aligned to 4 bytes\n");
return -EINVAL;
}
last_func_info_pos = hdr->hdr_len + hdr->func_info_off +
hdr->func_info_len;
if (last_func_info_pos > data_size) {
elog("Invalid BTF.ext func_info section size:%u\n",
hdr->func_info_len);
return -EINVAL;
}
finfo = data + hdr->hdr_len + hdr->func_info_off;
return btf_ext_validate_func_info(finfo, hdr->func_info_len,
err_log);
} }
void btf_ext__free(struct btf_ext *btf_ext) void btf_ext__free(struct btf_ext *btf_ext)
...@@ -601,16 +680,14 @@ void btf_ext__free(struct btf_ext *btf_ext) ...@@ -601,16 +680,14 @@ void btf_ext__free(struct btf_ext *btf_ext)
if (!btf_ext) if (!btf_ext)
return; return;
free(btf_ext->func_info); free(btf_ext->func_info.info);
free(btf_ext->line_info.info);
free(btf_ext); free(btf_ext);
} }
struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log)
{ {
const struct btf_ext_header *hdr;
struct btf_ext *btf_ext; struct btf_ext *btf_ext;
void *org_fdata, *fdata;
__u32 hdrlen, size_u32;
int err; int err;
err = btf_ext_parse_hdr(data, size, err_log); err = btf_ext_parse_hdr(data, size, err_log);
...@@ -621,42 +698,38 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) ...@@ -621,42 +698,38 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log)
if (!btf_ext) if (!btf_ext)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
hdr = (const struct btf_ext_header *)data; err = btf_ext_copy_func_info(btf_ext, data, size, err_log);
hdrlen = hdr->hdr_len; if (err) {
size_u32 = sizeof(__u32); btf_ext__free(btf_ext);
fdata = malloc(hdr->func_info_len - size_u32); return ERR_PTR(err);
if (!fdata) {
free(btf_ext);
return ERR_PTR(-ENOMEM);
} }
/* remember record size and copy rest of func_info data */ err = btf_ext_copy_line_info(btf_ext, data, size, err_log);
org_fdata = data + hdrlen + hdr->func_info_off; if (err) {
btf_ext->func_info_rec_size = *(__u32 *)org_fdata; btf_ext__free(btf_ext);
memcpy(fdata, org_fdata + size_u32, hdr->func_info_len - size_u32); return ERR_PTR(err);
btf_ext->func_info = fdata; }
btf_ext->func_info_len = hdr->func_info_len - size_u32;
return btf_ext; return btf_ext;
} }
int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, static int btf_ext_reloc_info(const struct btf *btf,
const char *sec_name, void **func_info, const struct btf_ext_info *ext_info,
__u32 *func_info_rec_size, __u32 *func_info_len) const char *sec_name, __u32 insns_cnt,
void **info, __u32 *cnt)
{ {
__u32 sec_hdrlen = sizeof(struct btf_sec_func_info); __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec);
__u32 i, record_size, records_len; __u32 i, record_size, existing_len, records_len;
struct btf_sec_func_info *sinfo; struct btf_ext_info_sec *sinfo;
const char *info_sec_name; const char *info_sec_name;
__s64 remain_len; __u64 remain_len;
void *data; void *data;
record_size = btf_ext->func_info_rec_size; record_size = ext_info->rec_size;
sinfo = btf_ext->func_info; sinfo = ext_info->info;
remain_len = btf_ext->func_info_len; remain_len = ext_info->len;
while (remain_len > 0) { while (remain_len > 0) {
records_len = sinfo->num_func_info * record_size; records_len = sinfo->num_info * record_size;
info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);
if (strcmp(info_sec_name, sec_name)) { if (strcmp(info_sec_name, sec_name)) {
remain_len -= sec_hdrlen + records_len; remain_len -= sec_hdrlen + records_len;
...@@ -664,79 +737,52 @@ int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, ...@@ -664,79 +737,52 @@ int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext,
continue; continue;
} }
data = malloc(records_len); existing_len = (*cnt) * record_size;
data = realloc(*info, existing_len + records_len);
if (!data) if (!data)
return -ENOMEM; return -ENOMEM;
memcpy(data, sinfo->data, records_len); memcpy(data + existing_len, sinfo->data, records_len);
/* adjust insn_off only, the rest data will be passed
/* adjust the insn_off, the data in .BTF.ext is * to the kernel.
* the actual byte offset, and the kernel expects
* the offset in term of bpf_insn.
*
* adjust the insn offset only, the rest data will
* be passed to kernel.
*/ */
for (i = 0; i < sinfo->num_func_info; i++) { for (i = 0; i < sinfo->num_info; i++) {
struct bpf_func_info_min *record; __u32 *insn_off;
record = data + i * record_size; insn_off = data + existing_len + (i * record_size);
record->insn_off /= sizeof(struct bpf_insn); *insn_off = *insn_off / sizeof(struct bpf_insn) +
insns_cnt;
} }
*info = data;
*func_info = data; *cnt += sinfo->num_info;
*func_info_len = records_len;
*func_info_rec_size = record_size;
return 0; return 0;
} }
return -EINVAL; return -ENOENT;
} }
int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, int btf_ext__reloc_func_info(const struct btf *btf, const struct btf_ext *btf_ext,
const char *sec_name, __u32 insns_cnt, const char *sec_name, __u32 insns_cnt,
void **func_info, __u32 *func_info_len) void **func_info, __u32 *cnt)
{ {
__u32 sec_hdrlen = sizeof(struct btf_sec_func_info); return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name,
__u32 i, record_size, existing_flen, records_len; insns_cnt, func_info, cnt);
struct btf_sec_func_info *sinfo; }
const char *info_sec_name;
__u64 remain_len;
void *data;
record_size = btf_ext->func_info_rec_size;
sinfo = btf_ext->func_info;
remain_len = btf_ext->func_info_len;
while (remain_len > 0) {
records_len = sinfo->num_func_info * record_size;
info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);
if (strcmp(info_sec_name, sec_name)) {
remain_len -= sec_hdrlen + records_len;
sinfo = (void *)sinfo + sec_hdrlen + records_len;
continue;
}
existing_flen = *func_info_len;
data = realloc(*func_info, existing_flen + records_len);
if (!data)
return -ENOMEM;
memcpy(data + existing_flen, sinfo->data, records_len); int btf_ext__reloc_line_info(const struct btf *btf, const struct btf_ext *btf_ext,
/* adjust insn_off only, the rest data will be passed const char *sec_name, __u32 insns_cnt,
* to the kernel. void **line_info, __u32 *cnt)
*/ {
for (i = 0; i < sinfo->num_func_info; i++) { return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name,
struct bpf_func_info_min *record; insns_cnt, line_info, cnt);
}
record = data + existing_flen + i * record_size; __u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext)
record->insn_off = {
record->insn_off / sizeof(struct bpf_insn) + return btf_ext->func_info.rec_size;
insns_cnt; }
}
*func_info = data;
*func_info_len = existing_flen + records_len;
return 0;
}
return -EINVAL; __u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext)
{
return btf_ext->line_info.rec_size;
} }
...@@ -51,13 +51,8 @@ struct btf_ext_header { ...@@ -51,13 +51,8 @@ struct btf_ext_header {
/* All offsets are in bytes relative to the end of this header */ /* All offsets are in bytes relative to the end of this header */
__u32 func_info_off; __u32 func_info_off;
__u32 func_info_len; __u32 func_info_len;
}; __u32 line_info_off;
__u32 line_info_len;
struct btf_sec_func_info {
__u32 sec_name_off;
__u32 num_func_info;
/* Followed by num_func_info number of bpf func_info records */
__u8 data[0];
}; };
typedef int (*btf_print_fn_t)(const char *, ...) typedef int (*btf_print_fn_t)(const char *, ...)
...@@ -77,12 +72,16 @@ LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf); ...@@ -77,12 +72,16 @@ LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);
struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log); struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log);
void btf_ext__free(struct btf_ext *btf_ext); void btf_ext__free(struct btf_ext *btf_ext);
int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, int btf_ext__reloc_func_info(const struct btf *btf,
const char *sec_name, void **func_info, const struct btf_ext *btf_ext,
__u32 *func_info_rec_size, __u32 *func_info_len); const char *sec_name, __u32 insns_cnt,
int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, void **func_info, __u32 *func_info_len);
const char *sec_name, __u32 insns_cnt, void **func_info, int btf_ext__reloc_line_info(const struct btf *btf,
__u32 *func_info_len); const struct btf_ext *btf_ext,
const char *sec_name, __u32 insns_cnt,
void **line_info, __u32 *cnt);
__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext);
__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext);
#ifdef __cplusplus #ifdef __cplusplus
} /* extern "C" */ } /* extern "C" */
......
...@@ -167,9 +167,13 @@ struct bpf_program { ...@@ -167,9 +167,13 @@ struct bpf_program {
int btf_fd; int btf_fd;
void *func_info; void *func_info;
__u32 func_info_rec_size; __u32 func_info_rec_size;
__u32 func_info_len; __u32 func_info_cnt;
struct bpf_capabilities *caps; struct bpf_capabilities *caps;
void *line_info;
__u32 line_info_rec_size;
__u32 line_info_cnt;
}; };
struct bpf_map { struct bpf_map {
...@@ -779,6 +783,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) ...@@ -779,6 +783,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
{ {
Elf *elf = obj->efile.elf; Elf *elf = obj->efile.elf;
GElf_Ehdr *ep = &obj->efile.ehdr; GElf_Ehdr *ep = &obj->efile.ehdr;
Elf_Data *btf_ext_data = NULL;
Elf_Scn *scn = NULL; Elf_Scn *scn = NULL;
int idx = 0, err = 0; int idx = 0, err = 0;
...@@ -841,14 +846,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) ...@@ -841,14 +846,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
obj->btf = NULL; obj->btf = NULL;
} }
} else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
obj->btf_ext = btf_ext__new(data->d_buf, data->d_size, btf_ext_data = data;
__pr_debug);
if (IS_ERR(obj->btf_ext)) {
pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
BTF_EXT_ELF_SEC,
PTR_ERR(obj->btf_ext));
obj->btf_ext = NULL;
}
} else if (sh.sh_type == SHT_SYMTAB) { } else if (sh.sh_type == SHT_SYMTAB) {
if (obj->efile.symbols) { if (obj->efile.symbols) {
pr_warning("bpf: multiple SYMTAB in %s\n", pr_warning("bpf: multiple SYMTAB in %s\n",
...@@ -910,6 +908,22 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) ...@@ -910,6 +908,22 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
pr_warning("Corrupted ELF file: index of strtab invalid\n"); pr_warning("Corrupted ELF file: index of strtab invalid\n");
return LIBBPF_ERRNO__FORMAT; return LIBBPF_ERRNO__FORMAT;
} }
if (btf_ext_data) {
if (!obj->btf) {
pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
BTF_EXT_ELF_SEC, BTF_ELF_SEC);
} else {
obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
btf_ext_data->d_size,
__pr_debug);
if (IS_ERR(obj->btf_ext)) {
pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
BTF_EXT_ELF_SEC,
PTR_ERR(obj->btf_ext));
obj->btf_ext = NULL;
}
}
}
if (obj->efile.maps_shndx >= 0) { if (obj->efile.maps_shndx >= 0) {
err = bpf_object__init_maps(obj, flags); err = bpf_object__init_maps(obj, flags);
if (err) if (err)
...@@ -1275,6 +1289,82 @@ bpf_object__create_maps(struct bpf_object *obj) ...@@ -1275,6 +1289,82 @@ bpf_object__create_maps(struct bpf_object *obj)
return 0; return 0;
} }
static int
check_btf_ext_reloc_err(struct bpf_program *prog, int err,
void *btf_prog_info, const char *info_name)
{
if (err != -ENOENT) {
pr_warning("Error in loading %s for sec %s.\n",
info_name, prog->section_name);
return err;
}
/* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */
if (btf_prog_info) {
/*
* Some info has already been found but has problem
* in the last btf_ext reloc. Must have to error
* out.
*/
pr_warning("Error in relocating %s for sec %s.\n",
info_name, prog->section_name);
return err;
}
/*
* Have problem loading the very first info. Ignore
* the rest.
*/
pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n",
info_name, prog->section_name, info_name);
return 0;
}
static int
bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj,
const char *section_name, __u32 insn_offset)
{
int err;
if (!insn_offset || prog->func_info) {
/*
* !insn_offset => main program
*
* For sub prog, the main program's func_info has to
* be loaded first (i.e. prog->func_info != NULL)
*/
err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext,
section_name, insn_offset,
&prog->func_info,
&prog->func_info_cnt);
if (err)
return check_btf_ext_reloc_err(prog, err,
prog->func_info,
"bpf_func_info");
prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext);
}
if (!insn_offset || prog->line_info) {
err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext,
section_name, insn_offset,
&prog->line_info,
&prog->line_info_cnt);
if (err)
return check_btf_ext_reloc_err(prog, err,
prog->line_info,
"bpf_line_info");
prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext);
}
if (!insn_offset)
prog->btf_fd = btf__fd(obj->btf);
return 0;
}
static int static int
bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
struct reloc_desc *relo) struct reloc_desc *relo)
...@@ -1306,17 +1396,12 @@ bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, ...@@ -1306,17 +1396,12 @@ bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
return -ENOMEM; return -ENOMEM;
} }
if (obj->btf && obj->btf_ext) { if (obj->btf_ext) {
err = btf_ext__reloc(obj->btf, obj->btf_ext, err = bpf_program_reloc_btf_ext(prog, obj,
text->section_name, text->section_name,
prog->insns_cnt, prog->insns_cnt);
&prog->func_info, if (err)
&prog->func_info_len);
if (err) {
pr_warning("error in btf_ext__reloc for sec %s\n",
text->section_name);
return err; return err;
}
} }
memcpy(new_insn + prog->insns_cnt, text->insns, memcpy(new_insn + prog->insns_cnt, text->insns,
...@@ -1341,18 +1426,11 @@ bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) ...@@ -1341,18 +1426,11 @@ bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
if (!prog) if (!prog)
return 0; return 0;
if (obj->btf && obj->btf_ext) { if (obj->btf_ext) {
err = btf_ext__reloc_init(obj->btf, obj->btf_ext, err = bpf_program_reloc_btf_ext(prog, obj,
prog->section_name, prog->section_name, 0);
&prog->func_info, if (err)
&prog->func_info_rec_size,
&prog->func_info_len);
if (err) {
pr_warning("err in btf_ext__reloc_init for sec %s\n",
prog->section_name);
return err; return err;
}
prog->btf_fd = btf__fd(obj->btf);
} }
if (!prog->reloc_desc) if (!prog->reloc_desc)
...@@ -1444,8 +1522,7 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) ...@@ -1444,8 +1522,7 @@ static int bpf_object__collect_reloc(struct bpf_object *obj)
static int static int
load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
char *license, __u32 kern_version, int *pfd, char *license, __u32 kern_version, int *pfd)
__u32 func_info_cnt)
{ {
struct bpf_load_program_attr load_attr; struct bpf_load_program_attr load_attr;
char *cp, errmsg[STRERR_BUFSIZE]; char *cp, errmsg[STRERR_BUFSIZE];
...@@ -1465,8 +1542,10 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, ...@@ -1465,8 +1542,10 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0;
load_attr.func_info = prog->func_info; load_attr.func_info = prog->func_info;
load_attr.func_info_rec_size = prog->func_info_rec_size; load_attr.func_info_rec_size = prog->func_info_rec_size;
load_attr.func_info_cnt = func_info_cnt; load_attr.func_info_cnt = prog->func_info_cnt;
load_attr.line_info = prog->line_info;
load_attr.line_info_rec_size = prog->line_info_rec_size;
load_attr.line_info_cnt = prog->line_info_cnt;
if (!load_attr.insns || !load_attr.insns_cnt) if (!load_attr.insns || !load_attr.insns_cnt)
return -EINVAL; return -EINVAL;
...@@ -1523,14 +1602,8 @@ int ...@@ -1523,14 +1602,8 @@ int
bpf_program__load(struct bpf_program *prog, bpf_program__load(struct bpf_program *prog,
char *license, __u32 kern_version) char *license, __u32 kern_version)
{ {
__u32 func_info_cnt;
int err = 0, fd, i; int err = 0, fd, i;
if (prog->func_info_len == 0)
func_info_cnt = 0;
else
func_info_cnt = prog->func_info_len / prog->func_info_rec_size;
if (prog->instances.nr < 0 || !prog->instances.fds) { if (prog->instances.nr < 0 || !prog->instances.fds) {
if (prog->preprocessor) { if (prog->preprocessor) {
pr_warning("Internal error: can't load program '%s'\n", pr_warning("Internal error: can't load program '%s'\n",
...@@ -1553,8 +1626,7 @@ bpf_program__load(struct bpf_program *prog, ...@@ -1553,8 +1626,7 @@ bpf_program__load(struct bpf_program *prog,
prog->section_name, prog->instances.nr); prog->section_name, prog->instances.nr);
} }
err = load_program(prog, prog->insns, prog->insns_cnt, err = load_program(prog, prog->insns, prog->insns_cnt,
license, kern_version, &fd, license, kern_version, &fd);
func_info_cnt);
if (!err) if (!err)
prog->instances.fds[0] = fd; prog->instances.fds[0] = fd;
goto out; goto out;
...@@ -1584,8 +1656,7 @@ bpf_program__load(struct bpf_program *prog, ...@@ -1584,8 +1656,7 @@ bpf_program__load(struct bpf_program *prog,
err = load_program(prog, result.new_insn_ptr, err = load_program(prog, result.new_insn_ptr,
result.new_insn_cnt, result.new_insn_cnt,
license, kern_version, &fd, license, kern_version, &fd);
func_info_cnt);
if (err) { if (err) {
pr_warning("Loading the %dth instance of program '%s' failed\n", pr_warning("Loading the %dth instance of program '%s' failed\n",
......
...@@ -342,6 +342,19 @@ int libbpf_nl_get_qdisc(int sock, unsigned int nl_pid, int ifindex, ...@@ -342,6 +342,19 @@ int libbpf_nl_get_qdisc(int sock, unsigned int nl_pid, int ifindex,
int libbpf_nl_get_filter(int sock, unsigned int nl_pid, int ifindex, int handle, int libbpf_nl_get_filter(int sock, unsigned int nl_pid, int ifindex, int handle,
libbpf_dump_nlmsg_t dump_filter_nlmsg, void *cookie); libbpf_dump_nlmsg_t dump_filter_nlmsg, void *cookie);
struct bpf_prog_linfo;
struct bpf_prog_info;
LIBBPF_API void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo);
LIBBPF_API struct bpf_prog_linfo *
bpf_prog_linfo__new(const struct bpf_prog_info *info);
LIBBPF_API const struct bpf_line_info *
bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo,
__u64 addr, __u32 func_idx, __u32 nr_skip);
LIBBPF_API const struct bpf_line_info *
bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo,
__u32 insn_off, __u32 nr_skip);
#ifdef __cplusplus #ifdef __cplusplus
} /* extern "C" */ } /* extern "C" */
#endif #endif
......
...@@ -99,6 +99,10 @@ LIBBPF_0.0.1 { ...@@ -99,6 +99,10 @@ LIBBPF_0.0.1 {
bpf_program__unload; bpf_program__unload;
bpf_program__unpin; bpf_program__unpin;
bpf_program__unpin_instance; bpf_program__unpin_instance;
bpf_prog_linfo__free;
bpf_prog_linfo__new;
bpf_prog_linfo__lfind_addr_func;
bpf_prog_linfo__lfind;
bpf_raw_tracepoint_open; bpf_raw_tracepoint_open;
bpf_set_link_xdp_fd; bpf_set_link_xdp_fd;
bpf_task_fd_query; bpf_task_fd_query;
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
#include <linux/err.h> #include <linux/err.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/filter.h> #include <linux/filter.h>
#include <linux/unistd.h>
#include <bpf/bpf.h> #include <bpf/bpf.h>
#include <sys/resource.h> #include <sys/resource.h>
#include <libelf.h> #include <libelf.h>
...@@ -107,19 +108,20 @@ static int __base_pr(const char *format, ...) ...@@ -107,19 +108,20 @@ static int __base_pr(const char *format, ...)
#define BTF_END_RAW 0xdeadbeef #define BTF_END_RAW 0xdeadbeef
#define NAME_TBD 0xdeadb33f #define NAME_TBD 0xdeadb33f
#define MAX_NR_RAW_TYPES 1024 #define MAX_NR_RAW_U32 1024
#define BTF_LOG_BUF_SIZE 65535 #define BTF_LOG_BUF_SIZE 65535
static struct args { static struct args {
unsigned int raw_test_num; unsigned int raw_test_num;
unsigned int file_test_num; unsigned int file_test_num;
unsigned int get_info_test_num; unsigned int get_info_test_num;
unsigned int info_raw_test_num;
bool raw_test; bool raw_test;
bool file_test; bool file_test;
bool get_info_test; bool get_info_test;
bool pprint_test; bool pprint_test;
bool always_log; bool always_log;
bool func_type_test; bool info_raw_test;
} args; } args;
static char btf_log_buf[BTF_LOG_BUF_SIZE]; static char btf_log_buf[BTF_LOG_BUF_SIZE];
...@@ -135,7 +137,7 @@ struct btf_raw_test { ...@@ -135,7 +137,7 @@ struct btf_raw_test {
const char *str_sec; const char *str_sec;
const char *map_name; const char *map_name;
const char *err_str; const char *err_str;
__u32 raw_types[MAX_NR_RAW_TYPES]; __u32 raw_types[MAX_NR_RAW_U32];
__u32 str_sec_size; __u32 str_sec_size;
enum bpf_map_type map_type; enum bpf_map_type map_type;
__u32 key_size; __u32 key_size;
...@@ -154,6 +156,9 @@ struct btf_raw_test { ...@@ -154,6 +156,9 @@ struct btf_raw_test {
int str_len_delta; int str_len_delta;
}; };
#define BTF_STR_SEC(str) \
.str_sec = str, .str_sec_size = sizeof(str)
static struct btf_raw_test raw_tests[] = { static struct btf_raw_test raw_tests[] = {
/* enum E { /* enum E {
* E0, * E0,
...@@ -1856,11 +1861,11 @@ static const char *get_next_str(const char *start, const char *end) ...@@ -1856,11 +1861,11 @@ static const char *get_next_str(const char *start, const char *end)
return start < end - 1 ? start + 1 : NULL; return start < end - 1 ? start + 1 : NULL;
} }
static int get_type_sec_size(const __u32 *raw_types) static int get_raw_sec_size(const __u32 *raw_types)
{ {
int i; int i;
for (i = MAX_NR_RAW_TYPES - 1; for (i = MAX_NR_RAW_U32 - 1;
i >= 0 && raw_types[i] != BTF_END_RAW; i >= 0 && raw_types[i] != BTF_END_RAW;
i--) i--)
; ;
...@@ -1872,7 +1877,8 @@ static void *btf_raw_create(const struct btf_header *hdr, ...@@ -1872,7 +1877,8 @@ static void *btf_raw_create(const struct btf_header *hdr,
const __u32 *raw_types, const __u32 *raw_types,
const char *str, const char *str,
unsigned int str_sec_size, unsigned int str_sec_size,
unsigned int *btf_size) unsigned int *btf_size,
const char **ret_next_str)
{ {
const char *next_str = str, *end_str = str + str_sec_size; const char *next_str = str, *end_str = str + str_sec_size;
unsigned int size_needed, offset; unsigned int size_needed, offset;
...@@ -1881,7 +1887,7 @@ static void *btf_raw_create(const struct btf_header *hdr, ...@@ -1881,7 +1887,7 @@ static void *btf_raw_create(const struct btf_header *hdr,
uint32_t *ret_types; uint32_t *ret_types;
void *raw_btf; void *raw_btf;
type_sec_size = get_type_sec_size(raw_types); type_sec_size = get_raw_sec_size(raw_types);
if (CHECK(type_sec_size < 0, "Cannot get nr_raw_types")) if (CHECK(type_sec_size < 0, "Cannot get nr_raw_types"))
return NULL; return NULL;
...@@ -1920,6 +1926,8 @@ static void *btf_raw_create(const struct btf_header *hdr, ...@@ -1920,6 +1926,8 @@ static void *btf_raw_create(const struct btf_header *hdr,
ret_hdr->str_len = str_sec_size; ret_hdr->str_len = str_sec_size;
*btf_size = size_needed; *btf_size = size_needed;
if (ret_next_str)
*ret_next_str = next_str;
return raw_btf; return raw_btf;
} }
...@@ -1939,7 +1947,7 @@ static int do_test_raw(unsigned int test_num) ...@@ -1939,7 +1947,7 @@ static int do_test_raw(unsigned int test_num)
test->raw_types, test->raw_types,
test->str_sec, test->str_sec,
test->str_sec_size, test->str_sec_size,
&raw_btf_size); &raw_btf_size, NULL);
if (!raw_btf) if (!raw_btf)
return -1; return -1;
...@@ -2016,7 +2024,7 @@ static int test_raw(void) ...@@ -2016,7 +2024,7 @@ static int test_raw(void)
struct btf_get_info_test { struct btf_get_info_test {
const char *descr; const char *descr;
const char *str_sec; const char *str_sec;
__u32 raw_types[MAX_NR_RAW_TYPES]; __u32 raw_types[MAX_NR_RAW_U32];
__u32 str_sec_size; __u32 str_sec_size;
int btf_size_delta; int btf_size_delta;
int (*special_test)(unsigned int test_num); int (*special_test)(unsigned int test_num);
...@@ -2096,7 +2104,7 @@ static int test_big_btf_info(unsigned int test_num) ...@@ -2096,7 +2104,7 @@ static int test_big_btf_info(unsigned int test_num)
test->raw_types, test->raw_types,
test->str_sec, test->str_sec,
test->str_sec_size, test->str_sec_size,
&raw_btf_size); &raw_btf_size, NULL);
if (!raw_btf) if (!raw_btf)
return -1; return -1;
...@@ -2180,7 +2188,7 @@ static int test_btf_id(unsigned int test_num) ...@@ -2180,7 +2188,7 @@ static int test_btf_id(unsigned int test_num)
test->raw_types, test->raw_types,
test->str_sec, test->str_sec,
test->str_sec_size, test->str_sec_size,
&raw_btf_size); &raw_btf_size, NULL);
if (!raw_btf) if (!raw_btf)
return -1; return -1;
...@@ -2318,7 +2326,7 @@ static int do_test_get_info(unsigned int test_num) ...@@ -2318,7 +2326,7 @@ static int do_test_get_info(unsigned int test_num)
test->raw_types, test->raw_types,
test->str_sec, test->str_sec,
test->str_sec_size, test->str_sec_size,
&raw_btf_size); &raw_btf_size, NULL);
if (!raw_btf) if (!raw_btf)
return -1; return -1;
...@@ -2847,7 +2855,7 @@ static int do_test_pprint(void) ...@@ -2847,7 +2855,7 @@ static int do_test_pprint(void)
fprintf(stderr, "%s......", test->descr); fprintf(stderr, "%s......", test->descr);
raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types, raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types,
test->str_sec, test->str_sec_size, test->str_sec, test->str_sec_size,
&raw_btf_size); &raw_btf_size, NULL);
if (!raw_btf) if (!raw_btf)
return -1; return -1;
...@@ -3051,18 +3059,25 @@ static int test_pprint(void) ...@@ -3051,18 +3059,25 @@ static int test_pprint(void)
return err; return err;
} }
static struct btf_func_type_test { #define BPF_LINE_INFO_ENC(insn_off, file_off, line_off, line_num, line_col) \
(insn_off), (file_off), (line_off), ((line_num) << 10 | ((line_col) & 0x3ff))
static struct prog_info_raw_test {
const char *descr; const char *descr;
const char *str_sec; const char *str_sec;
__u32 raw_types[MAX_NR_RAW_TYPES]; const char *err_str;
__u32 raw_types[MAX_NR_RAW_U32];
__u32 str_sec_size; __u32 str_sec_size;
struct bpf_insn insns[MAX_INSNS]; struct bpf_insn insns[MAX_INSNS];
__u32 prog_type; __u32 prog_type;
__u32 func_info[MAX_SUBPROGS][2]; __u32 func_info[MAX_SUBPROGS][2];
__u32 func_info_rec_size; __u32 func_info_rec_size;
__u32 func_info_cnt; __u32 func_info_cnt;
__u32 line_info[MAX_NR_RAW_U32];
__u32 line_info_rec_size;
__u32 nr_jited_ksyms;
bool expected_prog_load_failure; bool expected_prog_load_failure;
} func_type_test[] = { } info_raw_tests[] = {
{ {
.descr = "func_type (main func + one sub)", .descr = "func_type (main func + one sub)",
.raw_types = { .raw_types = {
...@@ -3091,6 +3106,7 @@ static struct btf_func_type_test { ...@@ -3091,6 +3106,7 @@ static struct btf_func_type_test {
.func_info = { {0, 5}, {3, 6} }, .func_info = { {0, 5}, {3, 6} },
.func_info_rec_size = 8, .func_info_rec_size = 8,
.func_info_cnt = 2, .func_info_cnt = 2,
.line_info = { BTF_END_RAW },
}, },
{ {
...@@ -3121,6 +3137,7 @@ static struct btf_func_type_test { ...@@ -3121,6 +3137,7 @@ static struct btf_func_type_test {
.func_info = { {0, 5}, {3, 6} }, .func_info = { {0, 5}, {3, 6} },
.func_info_rec_size = 4, .func_info_rec_size = 4,
.func_info_cnt = 2, .func_info_cnt = 2,
.line_info = { BTF_END_RAW },
.expected_prog_load_failure = true, .expected_prog_load_failure = true,
}, },
...@@ -3152,6 +3169,7 @@ static struct btf_func_type_test { ...@@ -3152,6 +3169,7 @@ static struct btf_func_type_test {
.func_info = { {0, 5}, {3, 6} }, .func_info = { {0, 5}, {3, 6} },
.func_info_rec_size = 8, .func_info_rec_size = 8,
.func_info_cnt = 1, .func_info_cnt = 1,
.line_info = { BTF_END_RAW },
.expected_prog_load_failure = true, .expected_prog_load_failure = true,
}, },
...@@ -3183,6 +3201,278 @@ static struct btf_func_type_test { ...@@ -3183,6 +3201,278 @@ static struct btf_func_type_test {
.func_info = { {0, 5}, {2, 6} }, .func_info = { {0, 5}, {2, 6} },
.func_info_rec_size = 8, .func_info_rec_size = 8,
.func_info_cnt = 2, .func_info_cnt = 2,
.line_info = { BTF_END_RAW },
.expected_prog_load_failure = true,
},
{
.descr = "line_info (No subprog)",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_END_RAW,
},
BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"),
.insns = {
BPF_MOV64_IMM(BPF_REG_0, 1),
BPF_MOV64_IMM(BPF_REG_1, 2),
BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.func_info_cnt = 0,
.line_info = {
BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9),
BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8),
BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7),
BTF_END_RAW,
},
.line_info_rec_size = sizeof(struct bpf_line_info),
.nr_jited_ksyms = 1,
},
{
.descr = "line_info (No subprog. insn_off >= prog->len)",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_END_RAW,
},
BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"),
.insns = {
BPF_MOV64_IMM(BPF_REG_0, 1),
BPF_MOV64_IMM(BPF_REG_1, 2),
BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.func_info_cnt = 0,
.line_info = {
BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9),
BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8),
BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7),
BPF_LINE_INFO_ENC(4, 0, 0, 5, 6),
BTF_END_RAW,
},
.line_info_rec_size = sizeof(struct bpf_line_info),
.nr_jited_ksyms = 1,
.err_str = "line_info[4].insn_off",
.expected_prog_load_failure = true,
},
{
.descr = "line_info (No subprog. zero tailing line_info",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_END_RAW,
},
BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"),
.insns = {
BPF_MOV64_IMM(BPF_REG_0, 1),
BPF_MOV64_IMM(BPF_REG_1, 2),
BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.func_info_cnt = 0,
.line_info = {
BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), 0,
BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9), 0,
BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8), 0,
BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7), 0,
BTF_END_RAW,
},
.line_info_rec_size = sizeof(struct bpf_line_info) + sizeof(__u32),
.nr_jited_ksyms = 1,
},
{
.descr = "line_info (No subprog. nonzero tailing line_info)",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_END_RAW,
},
BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"),
.insns = {
BPF_MOV64_IMM(BPF_REG_0, 1),
BPF_MOV64_IMM(BPF_REG_1, 2),
BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.func_info_cnt = 0,
.line_info = {
BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), 0,
BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9), 0,
BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8), 0,
BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7), 1,
BTF_END_RAW,
},
.line_info_rec_size = sizeof(struct bpf_line_info) + sizeof(__u32),
.nr_jited_ksyms = 1,
.err_str = "nonzero tailing record in line_info",
.expected_prog_load_failure = true,
},
{
.descr = "line_info (subprog)",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_END_RAW,
},
BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
.insns = {
BPF_MOV64_IMM(BPF_REG_2, 1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
BPF_CALL_REL(1),
BPF_EXIT_INSN(),
BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.func_info_cnt = 0,
.line_info = {
BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9),
BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 3, 8),
BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
BTF_END_RAW,
},
.line_info_rec_size = sizeof(struct bpf_line_info),
.nr_jited_ksyms = 2,
},
{
.descr = "line_info (subprog + func_info)",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_FUNC_PROTO_ENC(1, 1), /* [2] */
BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1),
BTF_FUNC_ENC(NAME_TBD, 2), /* [3] */
BTF_FUNC_ENC(NAME_TBD, 2), /* [4] */
BTF_END_RAW,
},
BTF_STR_SEC("\0int\0x\0sub\0main\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
.insns = {
BPF_MOV64_IMM(BPF_REG_2, 1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
BPF_CALL_REL(1),
BPF_EXIT_INSN(),
BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.func_info_cnt = 2,
.func_info_rec_size = 8,
.func_info = { {0, 4}, {5, 3} },
.line_info = {
BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9),
BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 3, 8),
BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
BTF_END_RAW,
},
.line_info_rec_size = sizeof(struct bpf_line_info),
.nr_jited_ksyms = 2,
},
{
.descr = "line_info (subprog. missing 1st func line info)",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_END_RAW,
},
BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
.insns = {
BPF_MOV64_IMM(BPF_REG_2, 1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
BPF_CALL_REL(1),
BPF_EXIT_INSN(),
BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.func_info_cnt = 0,
.line_info = {
BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 1, 10),
BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9),
BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 3, 8),
BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
BTF_END_RAW,
},
.line_info_rec_size = sizeof(struct bpf_line_info),
.nr_jited_ksyms = 2,
.err_str = "missing bpf_line_info for func#0",
.expected_prog_load_failure = true,
},
{
.descr = "line_info (subprog. missing 2nd func line info)",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_END_RAW,
},
BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
.insns = {
BPF_MOV64_IMM(BPF_REG_2, 1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
BPF_CALL_REL(1),
BPF_EXIT_INSN(),
BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.func_info_cnt = 0,
.line_info = {
BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9),
BPF_LINE_INFO_ENC(6, 0, NAME_TBD, 3, 8),
BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
BTF_END_RAW,
},
.line_info_rec_size = sizeof(struct bpf_line_info),
.nr_jited_ksyms = 2,
.err_str = "missing bpf_line_info for func#1",
.expected_prog_load_failure = true,
},
{
.descr = "line_info (subprog. unordered insn offset)",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_END_RAW,
},
BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
.insns = {
BPF_MOV64_IMM(BPF_REG_2, 1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
BPF_CALL_REL(1),
BPF_EXIT_INSN(),
BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.func_info_cnt = 0,
.line_info = {
BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 2, 9),
BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8),
BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
BTF_END_RAW,
},
.line_info_rec_size = sizeof(struct bpf_line_info),
.nr_jited_ksyms = 2,
.err_str = "Invalid line_info[2].insn_off",
.expected_prog_load_failure = true, .expected_prog_load_failure = true,
}, },
...@@ -3198,90 +3488,84 @@ static size_t probe_prog_length(const struct bpf_insn *fp) ...@@ -3198,90 +3488,84 @@ static size_t probe_prog_length(const struct bpf_insn *fp)
return len + 1; return len + 1;
} }
static int do_test_func_type(int test_num) static __u32 *patch_name_tbd(const __u32 *raw_u32,
const char *str, __u32 str_off,
unsigned int str_sec_size,
unsigned int *ret_size)
{ {
const struct btf_func_type_test *test = &func_type_test[test_num]; int i, raw_u32_size = get_raw_sec_size(raw_u32);
unsigned int raw_btf_size, info_len, rec_size; const char *end_str = str + str_sec_size;
int i, btf_fd = -1, prog_fd = -1, err = 0; const char *next_str = str + str_off;
struct bpf_load_program_attr attr = {}; __u32 *new_u32 = NULL;
void *raw_btf, *func_info = NULL;
struct bpf_prog_info info = {};
struct bpf_func_info *finfo;
fprintf(stderr, "%s......", test->descr); if (raw_u32_size == -1)
raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types, return ERR_PTR(-EINVAL);
test->str_sec, test->str_sec_size,
&raw_btf_size);
if (!raw_btf) if (!raw_u32_size) {
return -1; *ret_size = 0;
return NULL;
}
*btf_log_buf = '\0'; new_u32 = malloc(raw_u32_size);
btf_fd = bpf_load_btf(raw_btf, raw_btf_size, if (!new_u32)
btf_log_buf, BTF_LOG_BUF_SIZE, return ERR_PTR(-ENOMEM);
args.always_log);
free(raw_btf);
if (CHECK(btf_fd == -1, "invalid btf_fd errno:%d", errno)) { for (i = 0; i < raw_u32_size / sizeof(raw_u32[0]); i++) {
err = -1; if (raw_u32[i] == NAME_TBD) {
goto done; next_str = get_next_str(next_str, end_str);
if (CHECK(!next_str, "Error in getting next_str\n")) {
free(new_u32);
return ERR_PTR(-EINVAL);
}
new_u32[i] = next_str - str;
next_str += strlen(next_str);
} else {
new_u32[i] = raw_u32[i];
}
} }
if (*btf_log_buf && args.always_log) *ret_size = raw_u32_size;
fprintf(stderr, "\n%s", btf_log_buf); return new_u32;
}
attr.prog_type = test->prog_type;
attr.insns = test->insns;
attr.insns_cnt = probe_prog_length(attr.insns);
attr.license = "GPL";
attr.prog_btf_fd = btf_fd;
attr.func_info_rec_size = test->func_info_rec_size;
attr.func_info_cnt = test->func_info_cnt;
attr.func_info = test->func_info;
*btf_log_buf = '\0'; static int test_get_finfo(const struct prog_info_raw_test *test,
prog_fd = bpf_load_program_xattr(&attr, btf_log_buf, int prog_fd)
BTF_LOG_BUF_SIZE); {
if (test->expected_prog_load_failure && prog_fd == -1) { struct bpf_prog_info info = {};
err = 0; struct bpf_func_info *finfo;
goto done; __u32 info_len, rec_size, i;
} void *func_info = NULL;
if (CHECK(prog_fd == -1, "invalid prog_id errno:%d", errno)) { int err;
fprintf(stderr, "%s\n", btf_log_buf);
err = -1;
goto done;
}
/* get necessary lens */ /* get necessary lens */
info_len = sizeof(struct bpf_prog_info); info_len = sizeof(struct bpf_prog_info);
err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
if (CHECK(err == -1, "invalid get info (1st) errno:%d", errno)) { if (CHECK(err == -1, "invalid get info (1st) errno:%d", errno)) {
fprintf(stderr, "%s\n", btf_log_buf); fprintf(stderr, "%s\n", btf_log_buf);
err = -1; return -1;
goto done;
} }
if (CHECK(info.func_info_cnt != 2, if (CHECK(info.func_info_cnt != test->func_info_cnt,
"incorrect info.func_info_cnt (1st) %d\n", "incorrect info.func_info_cnt (1st) %d",
info.func_info_cnt)) { info.func_info_cnt)) {
err = -1; return -1;
goto done;
} }
rec_size = info.func_info_rec_size; rec_size = info.func_info_rec_size;
if (CHECK(rec_size < 4, if (CHECK(rec_size < 8,
"incorrect info.func_info_rec_size (1st) %d\n", rec_size)) { "incorrect info.func_info_rec_size (1st) %d", rec_size)) {
err = -1; return -1;
goto done;
} }
if (!info.func_info_cnt)
return 0;
func_info = malloc(info.func_info_cnt * rec_size); func_info = malloc(info.func_info_cnt * rec_size);
if (CHECK(!func_info, "out of memory")) { if (CHECK(!func_info, "out of memory"))
err = -1; return -1;
goto done;
}
/* reset info to only retrieve func_info related data */ /* reset info to only retrieve func_info related data */
memset(&info, 0, sizeof(info)); memset(&info, 0, sizeof(info));
info.func_info_cnt = 2; info.func_info_cnt = test->func_info_cnt;
info.func_info_rec_size = rec_size; info.func_info_rec_size = rec_size;
info.func_info = ptr_to_u64(func_info); info.func_info = ptr_to_u64(func_info);
err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
...@@ -3290,14 +3574,14 @@ static int do_test_func_type(int test_num) ...@@ -3290,14 +3574,14 @@ static int do_test_func_type(int test_num)
err = -1; err = -1;
goto done; goto done;
} }
if (CHECK(info.func_info_cnt != 2, if (CHECK(info.func_info_cnt != test->func_info_cnt,
"incorrect info.func_info_cnt (2nd) %d\n", "incorrect info.func_info_cnt (2nd) %d",
info.func_info_cnt)) { info.func_info_cnt)) {
err = -1; err = -1;
goto done; goto done;
} }
if (CHECK(info.func_info_rec_size != rec_size, if (CHECK(info.func_info_rec_size < 8,
"incorrect info.func_info_rec_size (2nd) %d\n", "incorrect info.func_info_rec_size (2nd) %d",
info.func_info_rec_size)) { info.func_info_rec_size)) {
err = -1; err = -1;
goto done; goto done;
...@@ -3310,7 +3594,7 @@ static int do_test_func_type(int test_num) ...@@ -3310,7 +3594,7 @@ static int do_test_func_type(int test_num)
} }
finfo = func_info; finfo = func_info;
for (i = 0; i < 2; i++) { for (i = 0; i < test->func_info_cnt; i++) {
if (CHECK(finfo->type_id != test->func_info[i][1], if (CHECK(finfo->type_id != test->func_info[i][1],
"incorrect func_type %u expected %u", "incorrect func_type %u expected %u",
finfo->type_id, test->func_info[i][1])) { finfo->type_id, test->func_info[i][1])) {
...@@ -3320,7 +3604,307 @@ static int do_test_func_type(int test_num) ...@@ -3320,7 +3604,307 @@ static int do_test_func_type(int test_num)
finfo = (void *)finfo + rec_size; finfo = (void *)finfo + rec_size;
} }
err = 0;
done:
free(func_info);
return err;
}
static int test_get_linfo(const struct prog_info_raw_test *test,
const void *patched_linfo,
__u32 cnt, int prog_fd)
{
__u32 i, info_len, nr_jited_ksyms, nr_jited_func_lens;
__u64 *jited_linfo = NULL, *jited_ksyms = NULL;
__u32 rec_size, jited_rec_size, jited_cnt;
struct bpf_line_info *linfo = NULL;
__u32 cur_func_len, ksyms_found;
struct bpf_prog_info info = {};
__u32 *jited_func_lens = NULL;
__u64 cur_func_ksyms;
int err;
jited_cnt = cnt;
rec_size = sizeof(*linfo);
jited_rec_size = sizeof(*jited_linfo);
if (test->nr_jited_ksyms)
nr_jited_ksyms = test->nr_jited_ksyms;
else
nr_jited_ksyms = test->func_info_cnt;
nr_jited_func_lens = nr_jited_ksyms;
info_len = sizeof(struct bpf_prog_info);
err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
if (CHECK(err == -1, "err:%d errno:%d", err, errno)) {
err = -1;
goto done;
}
if (!info.jited_prog_len) {
/* prog is not jited */
jited_cnt = 0;
nr_jited_ksyms = 1;
nr_jited_func_lens = 1;
}
if (CHECK(info.line_info_cnt != cnt ||
info.jited_line_info_cnt != jited_cnt ||
info.nr_jited_ksyms != nr_jited_ksyms ||
info.nr_jited_func_lens != nr_jited_func_lens ||
(!info.line_info_cnt && info.jited_line_info_cnt),
"info: line_info_cnt:%u(expected:%u) jited_line_info_cnt:%u(expected:%u) nr_jited_ksyms:%u(expected:%u) nr_jited_func_lens:%u(expected:%u)",
info.line_info_cnt, cnt,
info.jited_line_info_cnt, jited_cnt,
info.nr_jited_ksyms, nr_jited_ksyms,
info.nr_jited_func_lens, nr_jited_func_lens)) {
err = -1;
goto done;
}
if (CHECK(info.line_info_rec_size < 16 ||
info.jited_line_info_rec_size < 8,
"info: line_info_rec_size:%u(userspace expected:%u) jited_line_info_rec_size:%u(userspace expected:%u)",
info.line_info_rec_size, rec_size,
info.jited_line_info_rec_size, jited_rec_size)) {
err = -1;
goto done;
}
if (!cnt)
return 0;
rec_size = info.line_info_rec_size;
jited_rec_size = info.jited_line_info_rec_size;
memset(&info, 0, sizeof(info));
linfo = calloc(cnt, rec_size);
if (CHECK(!linfo, "!linfo")) {
err = -1;
goto done;
}
info.line_info_cnt = cnt;
info.line_info_rec_size = rec_size;
info.line_info = ptr_to_u64(linfo);
if (jited_cnt) {
jited_linfo = calloc(jited_cnt, jited_rec_size);
jited_ksyms = calloc(nr_jited_ksyms, sizeof(*jited_ksyms));
jited_func_lens = calloc(nr_jited_func_lens,
sizeof(*jited_func_lens));
if (CHECK(!jited_linfo || !jited_ksyms || !jited_func_lens,
"jited_linfo:%p jited_ksyms:%p jited_func_lens:%p",
jited_linfo, jited_ksyms, jited_func_lens)) {
err = -1;
goto done;
}
info.jited_line_info_cnt = jited_cnt;
info.jited_line_info_rec_size = jited_rec_size;
info.jited_line_info = ptr_to_u64(jited_linfo);
info.nr_jited_ksyms = nr_jited_ksyms;
info.jited_ksyms = ptr_to_u64(jited_ksyms);
info.nr_jited_func_lens = nr_jited_func_lens;
info.jited_func_lens = ptr_to_u64(jited_func_lens);
}
err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
/*
* Only recheck the info.*line_info* fields.
* Other fields are not the concern of this test.
*/
if (CHECK(err == -1 ||
!info.line_info ||
info.line_info_cnt != cnt ||
(jited_cnt && !info.jited_line_info) ||
info.jited_line_info_cnt != jited_cnt ||
info.line_info_rec_size != rec_size ||
info.jited_line_info_rec_size != jited_rec_size,
"err:%d errno:%d info: line_info_cnt:%u(expected:%u) jited_line_info_cnt:%u(expected:%u) line_info_rec_size:%u(expected:%u) jited_linfo_rec_size:%u(expected:%u) line_info:%p jited_line_info:%p",
err, errno,
info.line_info_cnt, cnt,
info.jited_line_info_cnt, jited_cnt,
info.line_info_rec_size, rec_size,
info.jited_line_info_rec_size, jited_rec_size,
(void *)(long)info.line_info,
(void *)(long)info.jited_line_info)) {
err = -1;
goto done;
}
CHECK(linfo[0].insn_off, "linfo[0].insn_off:%u",
linfo[0].insn_off);
for (i = 1; i < cnt; i++) {
const struct bpf_line_info *expected_linfo;
expected_linfo = patched_linfo + (i * test->line_info_rec_size);
if (CHECK(linfo[i].insn_off <= linfo[i - 1].insn_off,
"linfo[%u].insn_off:%u <= linfo[%u].insn_off:%u",
i, linfo[i].insn_off,
i - 1, linfo[i - 1].insn_off)) {
err = -1;
goto done;
}
if (CHECK(linfo[i].file_name_off != expected_linfo->file_name_off ||
linfo[i].line_off != expected_linfo->line_off ||
linfo[i].line_col != expected_linfo->line_col,
"linfo[%u] (%u, %u, %u) != (%u, %u, %u)", i,
linfo[i].file_name_off,
linfo[i].line_off,
linfo[i].line_col,
expected_linfo->file_name_off,
expected_linfo->line_off,
expected_linfo->line_col)) {
err = -1;
goto done;
}
}
if (!jited_cnt) {
fprintf(stderr, "not jited. skipping jited_line_info check. ");
err = 0;
goto done;
}
if (CHECK(jited_linfo[0] != jited_ksyms[0],
"jited_linfo[0]:%lx != jited_ksyms[0]:%lx",
(long)(jited_linfo[0]), (long)(jited_ksyms[0]))) {
err = -1;
goto done;
}
ksyms_found = 1;
cur_func_len = jited_func_lens[0];
cur_func_ksyms = jited_ksyms[0];
for (i = 1; i < jited_cnt; i++) {
if (ksyms_found < nr_jited_ksyms &&
jited_linfo[i] == jited_ksyms[ksyms_found]) {
cur_func_ksyms = jited_ksyms[ksyms_found];
cur_func_len = jited_ksyms[ksyms_found];
ksyms_found++;
continue;
}
if (CHECK(jited_linfo[i] <= jited_linfo[i - 1],
"jited_linfo[%u]:%lx <= jited_linfo[%u]:%lx",
i, (long)jited_linfo[i],
i - 1, (long)(jited_linfo[i - 1]))) {
err = -1;
goto done;
}
if (CHECK(jited_linfo[i] - cur_func_ksyms > cur_func_len,
"jited_linfo[%u]:%lx - %lx > %u",
i, (long)jited_linfo[i], (long)cur_func_ksyms,
cur_func_len)) {
err = -1;
goto done;
}
}
if (CHECK(ksyms_found != nr_jited_ksyms,
"ksyms_found:%u != nr_jited_ksyms:%u",
ksyms_found, nr_jited_ksyms)) {
err = -1;
goto done;
}
err = 0;
done: done:
free(linfo);
free(jited_linfo);
free(jited_ksyms);
free(jited_func_lens);
return err;
}
static int do_test_info_raw(unsigned int test_num)
{
const struct prog_info_raw_test *test = &info_raw_tests[test_num - 1];
unsigned int raw_btf_size, linfo_str_off, linfo_size;
int btf_fd = -1, prog_fd = -1, err = 0;
void *raw_btf, *patched_linfo = NULL;
const char *ret_next_str;
union bpf_attr attr = {};
fprintf(stderr, "BTF prog info raw test[%u] (%s): ", test_num, test->descr);
raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types,
test->str_sec, test->str_sec_size,
&raw_btf_size, &ret_next_str);
if (!raw_btf)
return -1;
*btf_log_buf = '\0';
btf_fd = bpf_load_btf(raw_btf, raw_btf_size,
btf_log_buf, BTF_LOG_BUF_SIZE,
args.always_log);
free(raw_btf);
if (CHECK(btf_fd == -1, "invalid btf_fd errno:%d", errno)) {
err = -1;
goto done;
}
if (*btf_log_buf && args.always_log)
fprintf(stderr, "\n%s", btf_log_buf);
*btf_log_buf = '\0';
linfo_str_off = ret_next_str - test->str_sec;
patched_linfo = patch_name_tbd(test->line_info,
test->str_sec, linfo_str_off,
test->str_sec_size, &linfo_size);
if (IS_ERR(patched_linfo)) {
fprintf(stderr, "error in creating raw bpf_line_info");
err = -1;
goto done;
}
attr.prog_type = test->prog_type;
attr.insns = ptr_to_u64(test->insns);
attr.insn_cnt = probe_prog_length(test->insns);
attr.license = ptr_to_u64("GPL");
attr.prog_btf_fd = btf_fd;
attr.func_info_rec_size = test->func_info_rec_size;
attr.func_info_cnt = test->func_info_cnt;
attr.func_info = ptr_to_u64(test->func_info);
attr.log_buf = ptr_to_u64(btf_log_buf);
attr.log_size = BTF_LOG_BUF_SIZE;
attr.log_level = 1;
if (linfo_size) {
attr.line_info_rec_size = test->line_info_rec_size;
attr.line_info = ptr_to_u64(patched_linfo);
attr.line_info_cnt = linfo_size / attr.line_info_rec_size;
}
prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr));
err = ((prog_fd == -1) != test->expected_prog_load_failure);
if (CHECK(err, "prog_fd:%d expected_prog_load_failure:%u errno:%d",
prog_fd, test->expected_prog_load_failure, errno) ||
CHECK(test->err_str && !strstr(btf_log_buf, test->err_str),
"expected err_str:%s", test->err_str)) {
err = -1;
goto done;
}
if (prog_fd == -1)
goto done;
err = test_get_finfo(test, prog_fd);
if (err)
goto done;
err = test_get_linfo(test, patched_linfo, attr.line_info_cnt, prog_fd);
if (err)
goto done;
done:
if (!err)
fprintf(stderr, "OK");
if (*btf_log_buf && (err || args.always_log)) if (*btf_log_buf && (err || args.always_log))
fprintf(stderr, "\n%s", btf_log_buf); fprintf(stderr, "\n%s", btf_log_buf);
...@@ -3328,33 +3912,41 @@ static int do_test_func_type(int test_num) ...@@ -3328,33 +3912,41 @@ static int do_test_func_type(int test_num)
close(btf_fd); close(btf_fd);
if (prog_fd != -1) if (prog_fd != -1)
close(prog_fd); close(prog_fd);
free(func_info);
if (!IS_ERR(patched_linfo))
free(patched_linfo);
return err; return err;
} }
static int test_func_type(void) static int test_info_raw(void)
{ {
unsigned int i; unsigned int i;
int err = 0; int err = 0;
for (i = 0; i < ARRAY_SIZE(func_type_test); i++) if (args.info_raw_test_num)
err |= count_result(do_test_func_type(i)); return count_result(do_test_info_raw(args.info_raw_test_num));
for (i = 1; i <= ARRAY_SIZE(info_raw_tests); i++)
err |= count_result(do_test_info_raw(i));
return err; return err;
} }
static void usage(const char *cmd) static void usage(const char *cmd)
{ {
fprintf(stderr, "Usage: %s [-l] [[-r test_num (1 - %zu)] |" fprintf(stderr, "Usage: %s [-l] [[-r btf_raw_test_num (1 - %zu)] |\n"
" [-g test_num (1 - %zu)] |" "\t[-g btf_get_info_test_num (1 - %zu)] |\n"
" [-f test_num (1 - %zu)] | [-p] | [-k] ]\n", "\t[-f btf_file_test_num (1 - %zu)] |\n"
"\t[-k btf_prog_info_raw_test_num (1 - %zu)] |\n"
"\t[-p (pretty print test)]]\n",
cmd, ARRAY_SIZE(raw_tests), ARRAY_SIZE(get_info_tests), cmd, ARRAY_SIZE(raw_tests), ARRAY_SIZE(get_info_tests),
ARRAY_SIZE(file_tests)); ARRAY_SIZE(file_tests), ARRAY_SIZE(info_raw_tests));
} }
static int parse_args(int argc, char **argv) static int parse_args(int argc, char **argv)
{ {
const char *optstr = "lpkf:r:g:"; const char *optstr = "lpk:f:r:g:";
int opt; int opt;
while ((opt = getopt(argc, argv, optstr)) != -1) { while ((opt = getopt(argc, argv, optstr)) != -1) {
...@@ -3378,7 +3970,8 @@ static int parse_args(int argc, char **argv) ...@@ -3378,7 +3970,8 @@ static int parse_args(int argc, char **argv)
args.pprint_test = true; args.pprint_test = true;
break; break;
case 'k': case 'k':
args.func_type_test = true; args.info_raw_test_num = atoi(optarg);
args.info_raw_test = true;
break; break;
case 'h': case 'h':
usage(argv[0]); usage(argv[0]);
...@@ -3413,6 +4006,14 @@ static int parse_args(int argc, char **argv) ...@@ -3413,6 +4006,14 @@ static int parse_args(int argc, char **argv)
return -1; return -1;
} }
if (args.info_raw_test_num &&
(args.info_raw_test_num < 1 ||
args.info_raw_test_num > ARRAY_SIZE(info_raw_tests))) {
fprintf(stderr, "BTF prog info raw test number must be [1 - %zu]\n",
ARRAY_SIZE(info_raw_tests));
return -1;
}
return 0; return 0;
} }
...@@ -3445,16 +4046,17 @@ int main(int argc, char **argv) ...@@ -3445,16 +4046,17 @@ int main(int argc, char **argv)
if (args.pprint_test) if (args.pprint_test)
err |= test_pprint(); err |= test_pprint();
if (args.func_type_test) if (args.info_raw_test)
err |= test_func_type(); err |= test_info_raw();
if (args.raw_test || args.get_info_test || args.file_test || if (args.raw_test || args.get_info_test || args.file_test ||
args.pprint_test || args.func_type_test) args.pprint_test || args.info_raw_test)
goto done; goto done;
err |= test_raw(); err |= test_raw();
err |= test_get_info(); err |= test_get_info();
err |= test_file(); err |= test_file();
err |= test_info_raw();
done: done:
print_summary(); print_summary();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment