Commit b65053cd authored by Andrii Nakryiko's avatar Andrii Nakryiko Committed by Alexei Starovoitov

selftests/bpf: Add whitelist/blacklist of test names to test_progs

Add ability to specify a list of test name substrings for selecting which
tests to run. So now -t is accepting a comma-separated list of strings,
similarly to how -n accepts a comma-separated list of test numbers.

Additionally, add ability to blacklist tests by name. Blacklist takes
precedence over whitelist. Blacklisting is important for cases where it's
known that some tests can't pass (e.g., due to perf hardware events that are
not available within VM). This is going to be used for libbpf testing in
Travis CI in its Github repo.

Example runs with just whitelist and whitelist + blacklist:

  $ sudo ./test_progs -tattach,core/existence
  #1 attach_probe:OK
  #6 cgroup_attach_autodetach:OK
  #7 cgroup_attach_multi:OK
  #8 cgroup_attach_override:OK
  #9 core_extern:OK
  #10/44 existence:OK
  #10/45 existence___minimal:OK
  #10/46 existence__err_int_sz:OK
  #10/47 existence__err_int_type:OK
  #10/48 existence__err_int_kind:OK
  #10/49 existence__err_arr_kind:OK
  #10/50 existence__err_arr_value_type:OK
  #10/51 existence__err_struct_type:OK
  #10 core_reloc:OK
  #19 flow_dissector_reattach:OK
  #60 tp_attach_query:OK
  Summary: 8/8 PASSED, 0 SKIPPED, 0 FAILED

  $ sudo ./test_progs -tattach,core/existence -bcgroup,flow/arr
  #1 attach_probe:OK
  #9 core_extern:OK
  #10/44 existence:OK
  #10/45 existence___minimal:OK
  #10/46 existence__err_int_sz:OK
  #10/47 existence__err_int_type:OK
  #10/48 existence__err_int_kind:OK
  #10/51 existence__err_struct_type:OK
  #10 core_reloc:OK
  #60 tp_attach_query:OK
  Summary: 4/6 PASSED, 0 SKIPPED, 0 FAILED
Signed-off-by: default avatarAndrii Nakryiko <andriin@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Cc: Julia Kartseva <hex@fb.com>
Link: https://lore.kernel.org/bpf/20200116005549.3644118-1-andriin@fb.com
parent 7bcfea96
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
#include <string.h> #include <string.h>
/* defined in test_progs.h */ /* defined in test_progs.h */
struct test_env env; struct test_env env = {};
struct prog_test_def { struct prog_test_def {
const char *test_name; const char *test_name;
...@@ -29,10 +29,19 @@ struct prog_test_def { ...@@ -29,10 +29,19 @@ struct prog_test_def {
static bool should_run(struct test_selector *sel, int num, const char *name) static bool should_run(struct test_selector *sel, int num, const char *name)
{ {
if (sel->name && sel->name[0] && !strstr(name, sel->name)) int i;
return false;
for (i = 0; i < sel->blacklist.cnt; i++) {
if (strstr(name, sel->blacklist.strs[i]))
return false;
}
if (!sel->num_set) for (i = 0; i < sel->whitelist.cnt; i++) {
if (strstr(name, sel->whitelist.strs[i]))
return true;
}
if (!sel->whitelist.cnt && !sel->num_set)
return true; return true;
return num < sel->num_set_len && sel->num_set[num]; return num < sel->num_set_len && sel->num_set[num];
...@@ -334,6 +343,7 @@ const char argp_program_doc[] = "BPF selftests test runner"; ...@@ -334,6 +343,7 @@ const char argp_program_doc[] = "BPF selftests test runner";
enum ARG_KEYS { enum ARG_KEYS {
ARG_TEST_NUM = 'n', ARG_TEST_NUM = 'n',
ARG_TEST_NAME = 't', ARG_TEST_NAME = 't',
ARG_TEST_NAME_BLACKLIST = 'b',
ARG_VERIFIER_STATS = 's', ARG_VERIFIER_STATS = 's',
ARG_VERBOSE = 'v', ARG_VERBOSE = 'v',
}; };
...@@ -341,8 +351,10 @@ enum ARG_KEYS { ...@@ -341,8 +351,10 @@ enum ARG_KEYS {
static const struct argp_option opts[] = { static const struct argp_option opts[] = {
{ "num", ARG_TEST_NUM, "NUM", 0, { "num", ARG_TEST_NUM, "NUM", 0,
"Run test number NUM only " }, "Run test number NUM only " },
{ "name", ARG_TEST_NAME, "NAME", 0, { "name", ARG_TEST_NAME, "NAMES", 0,
"Run tests with names containing NAME" }, "Run tests with names containing any string from NAMES list" },
{ "name-blacklist", ARG_TEST_NAME_BLACKLIST, "NAMES", 0,
"Don't run tests with names containing any string from NAMES list" },
{ "verifier-stats", ARG_VERIFIER_STATS, NULL, 0, { "verifier-stats", ARG_VERIFIER_STATS, NULL, 0,
"Output verifier statistics", }, "Output verifier statistics", },
{ "verbose", ARG_VERBOSE, "LEVEL", OPTION_ARG_OPTIONAL, { "verbose", ARG_VERBOSE, "LEVEL", OPTION_ARG_OPTIONAL,
...@@ -359,6 +371,41 @@ static int libbpf_print_fn(enum libbpf_print_level level, ...@@ -359,6 +371,41 @@ static int libbpf_print_fn(enum libbpf_print_level level,
return 0; return 0;
} }
static int parse_str_list(const char *s, struct str_set *set)
{
char *input, *state = NULL, *next, **tmp, **strs = NULL;
int cnt = 0;
input = strdup(s);
if (!input)
return -ENOMEM;
set->cnt = 0;
set->strs = NULL;
while ((next = strtok_r(state ? NULL : input, ",", &state))) {
tmp = realloc(strs, sizeof(*strs) * (cnt + 1));
if (!tmp)
goto err;
strs = tmp;
strs[cnt] = strdup(next);
if (!strs[cnt])
goto err;
cnt++;
}
set->cnt = cnt;
set->strs = (const char **)strs;
free(input);
return 0;
err:
free(strs);
free(input);
return -ENOMEM;
}
int parse_num_list(const char *s, struct test_selector *sel) int parse_num_list(const char *s, struct test_selector *sel)
{ {
int i, set_len = 0, num, start = 0, end = -1; int i, set_len = 0, num, start = 0, end = -1;
...@@ -449,12 +496,24 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) ...@@ -449,12 +496,24 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state)
if (subtest_str) { if (subtest_str) {
*subtest_str = '\0'; *subtest_str = '\0';
env->subtest_selector.name = strdup(subtest_str + 1); if (parse_str_list(subtest_str + 1,
if (!env->subtest_selector.name) &env->subtest_selector.whitelist))
return -ENOMEM;
}
if (parse_str_list(arg, &env->test_selector.whitelist))
return -ENOMEM;
break;
}
case ARG_TEST_NAME_BLACKLIST: {
char *subtest_str = strchr(arg, '/');
if (subtest_str) {
*subtest_str = '\0';
if (parse_str_list(subtest_str + 1,
&env->subtest_selector.blacklist))
return -ENOMEM; return -ENOMEM;
} }
env->test_selector.name = strdup(arg); if (parse_str_list(arg, &env->test_selector.blacklist))
if (!env->test_selector.name)
return -ENOMEM; return -ENOMEM;
break; break;
} }
...@@ -617,7 +676,11 @@ int main(int argc, char **argv) ...@@ -617,7 +676,11 @@ int main(int argc, char **argv)
printf("Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n", printf("Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n",
env.succ_cnt, env.sub_succ_cnt, env.skip_cnt, env.fail_cnt); env.succ_cnt, env.sub_succ_cnt, env.skip_cnt, env.fail_cnt);
free(env.test_selector.blacklist.strs);
free(env.test_selector.whitelist.strs);
free(env.test_selector.num_set); free(env.test_selector.num_set);
free(env.subtest_selector.blacklist.strs);
free(env.subtest_selector.whitelist.strs);
free(env.subtest_selector.num_set); free(env.subtest_selector.num_set);
return env.fail_cnt ? EXIT_FAILURE : EXIT_SUCCESS; return env.fail_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
......
...@@ -46,8 +46,14 @@ enum verbosity { ...@@ -46,8 +46,14 @@ enum verbosity {
VERBOSE_SUPER, VERBOSE_SUPER,
}; };
struct str_set {
const char **strs;
int cnt;
};
struct test_selector { struct test_selector {
const char *name; struct str_set whitelist;
struct str_set blacklist;
bool *num_set; bool *num_set;
int num_set_len; int num_set_len;
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment