Commit 61b75948 authored by Alexei Starovoitov's avatar Alexei Starovoitov

Merge branch 'libbpf: add support for privileged/unprivileged control separation'

Mariusz Dudek says:

====================

From: Mariusz Dudek <mariuszx.dudek@intel.com>

This patch series adds support for separation of eBPF program
load and xsk socket creation. In for example a Kubernetes
environment you can have an AF_XDP CNI or daemonset that is
responsible for launching pods that execute an application
using AF_XDP sockets. It is desirable that the pod runs with
as low privileges as possible, CAP_NET_RAW in this case,
and that all operations that require privileges are contained
in the CNI or daemonset.

In this case, you have to be able separate ePBF program load from
xsk socket creation.

Currently, this will not work with the xsk_socket__create APIs
because you need to have CAP_NET_ADMIN privileges to load eBPF
program and CAP_SYS_ADMIN privileges to create update xsk_bpf_maps.
To be exact xsk_set_bpf_maps does not need those privileges but
it takes the prog_fd and xsks_map_fd and those are known only to
process that was loading eBPF program. The api bpf_prog_get_fd_by_id
that looks up the fd of the prog using an prog_id and
bpf_map_get_fd_by_id that looks for xsks_map_fd usinb map_id both
requires CAP_SYS_ADMIN.

With this patch, the pod can be run with CAP_NET_RAW capability
only. In case your umem is larger or equal process limit for
MEMLOCK you need either increase the limit or CAP_IPC_LOCK capability.
Without this patch in case of insufficient rights ENOPERM is
returned by xsk_socket__create.

To resolve this privileges issue two new APIs are introduced:
- xsk_setup_xdp_prog - loads the built in XDP program. It can
also return xsks_map_fd which is needed by unprivileged
process to update xsks_map with AF_XDP socket "fd"
- xsk_sokcet__update_xskmap - inserts an AF_XDP socket into an
xskmap for a particular xsk_socket

Usage example:
int xsk_setup_xdp_prog(int ifindex, int *xsks_map_fd)

int xsk_socket__update_xskmap(struct xsk_socket *xsk, int xsks_map_fd);

Inserts AF_XDP socket "fd" into the xskmap.

The first patch introduces the new APIs. The second patch provides
a new sample applications working as control and modification to
existing xdpsock application to work with less privileges.

This patch set is based on bpf-next commit 97306be4
(Merge branch 'switch to memcg-based memory accounting')

Since v6
- rebase on 97306be4 to resolve RLIMIT conflicts

Since v5
- fixed sample/bpf/xdpsock_user.c to resolve merge conflicts

Since v4
- sample/bpf/Makefile issues fixed

Since v3:
- force_set_map flag removed
- leaking of xsk struct fixed
- unified function error returning policy implemented

Since v2:
- new APIs moved itto LIBBPF_0.3.0 section
- struct bpf_prog_cfg_opts removed
- loading own eBPF program via xsk_setup_xdp_prog functionality removed

Since v1:
- struct bpf_prog_cfg improved for backward/forward compatibility
- API xsk_update_xskmap renamed to xsk_socket__update_xskmap
- commit message formatting fixed
====================
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 22e8ebe3 3627d970
......@@ -48,6 +48,7 @@ tprogs-y += syscall_tp
tprogs-y += cpustat
tprogs-y += xdp_adjust_tail
tprogs-y += xdpsock
tprogs-y += xdpsock_ctrl_proc
tprogs-y += xsk_fwd
tprogs-y += xdp_fwd
tprogs-y += task_fd_query
......@@ -105,6 +106,7 @@ syscall_tp-objs := syscall_tp_user.o
cpustat-objs := cpustat_user.o
xdp_adjust_tail-objs := xdp_adjust_tail_user.o
xdpsock-objs := xdpsock_user.o
xdpsock_ctrl_proc-objs := xdpsock_ctrl_proc.o
xsk_fwd-objs := xsk_fwd.o
xdp_fwd-objs := xdp_fwd_user.o
task_fd_query-objs := task_fd_query_user.o $(TRACE_HELPERS)
......@@ -202,7 +204,7 @@ TPROGLDLIBS_tracex4 += -lrt
TPROGLDLIBS_trace_output += -lrt
TPROGLDLIBS_map_perf_test += -lrt
TPROGLDLIBS_test_overhead += -lrt
TPROGLDLIBS_xdpsock += -pthread
TPROGLDLIBS_xdpsock += -pthread -lcap
TPROGLDLIBS_xsk_fwd += -pthread
# Allows pointing LLC/CLANG to a LLVM backend with bpf support, redefine on cmdline:
......
......@@ -8,4 +8,12 @@
#define MAX_SOCKS 4
#define SOCKET_NAME "sock_cal_bpf_fd"
#define MAX_NUM_OF_CLIENTS 10
#define CLOSE_CONN 1
typedef __u64 u64;
typedef __u32 u32;
#endif /* XDPSOCK_H */
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2017 - 2018 Intel Corporation. */
#include <errno.h>
#include <getopt.h>
#include <libgen.h>
#include <net/if.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <unistd.h>
#include <bpf/bpf.h>
#include <bpf/xsk.h>
#include "xdpsock.h"
static const char *opt_if = "";
static struct option long_options[] = {
{"interface", required_argument, 0, 'i'},
{0, 0, 0, 0}
};
static void usage(const char *prog)
{
const char *str =
" Usage: %s [OPTIONS]\n"
" Options:\n"
" -i, --interface=n Run on interface n\n"
"\n";
fprintf(stderr, "%s\n", str);
exit(0);
}
static void parse_command_line(int argc, char **argv)
{
int option_index, c;
opterr = 0;
for (;;) {
c = getopt_long(argc, argv, "i:",
long_options, &option_index);
if (c == -1)
break;
switch (c) {
case 'i':
opt_if = optarg;
break;
default:
usage(basename(argv[0]));
}
}
}
static int send_xsks_map_fd(int sock, int fd)
{
char cmsgbuf[CMSG_SPACE(sizeof(int))];
struct msghdr msg;
struct iovec iov;
int value = 0;
if (fd == -1) {
fprintf(stderr, "Incorrect fd = %d\n", fd);
return -1;
}
iov.iov_base = &value;
iov.iov_len = sizeof(int);
msg.msg_name = NULL;
msg.msg_namelen = 0;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
msg.msg_flags = 0;
msg.msg_control = cmsgbuf;
msg.msg_controllen = CMSG_LEN(sizeof(int));
struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
*(int *)CMSG_DATA(cmsg) = fd;
int ret = sendmsg(sock, &msg, 0);
if (ret == -1) {
fprintf(stderr, "Sendmsg failed with %s", strerror(errno));
return -errno;
}
return ret;
}
int
main(int argc, char **argv)
{
struct sockaddr_un server;
int listening = 1;
int rval, msgsock;
int ifindex = 0;
int flag = 1;
int cmd = 0;
int sock;
int err;
int xsks_map_fd;
parse_command_line(argc, argv);
ifindex = if_nametoindex(opt_if);
if (ifindex == 0) {
fprintf(stderr, "Unable to get ifindex for Interface %s. Reason:%s",
opt_if, strerror(errno));
return -errno;
}
sock = socket(AF_UNIX, SOCK_STREAM, 0);
if (sock < 0) {
fprintf(stderr, "Opening socket stream failed: %s", strerror(errno));
return -errno;
}
server.sun_family = AF_UNIX;
strcpy(server.sun_path, SOCKET_NAME);
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(int));
if (bind(sock, (struct sockaddr *)&server, sizeof(struct sockaddr_un))) {
fprintf(stderr, "Binding to socket stream failed: %s", strerror(errno));
return -errno;
}
listen(sock, MAX_NUM_OF_CLIENTS);
err = xsk_setup_xdp_prog(ifindex, &xsks_map_fd);
if (err) {
fprintf(stderr, "Setup of xdp program failed\n");
goto close_sock;
}
while (listening) {
msgsock = accept(sock, 0, 0);
if (msgsock == -1) {
fprintf(stderr, "Error accepting connection: %s", strerror(errno));
err = -errno;
goto close_sock;
}
err = send_xsks_map_fd(msgsock, xsks_map_fd);
if (err <= 0) {
fprintf(stderr, "Error %d sending xsks_map_fd\n", err);
goto cleanup;
}
do {
rval = read(msgsock, &cmd, sizeof(int));
if (rval < 0) {
fprintf(stderr, "Error reading stream message");
} else {
if (cmd != CLOSE_CONN)
fprintf(stderr, "Recv unknown cmd = %d\n", cmd);
listening = 0;
break;
}
} while (rval > 0);
}
close(msgsock);
close(sock);
unlink(SOCKET_NAME);
/* Unset fd for given ifindex */
err = bpf_set_link_xdp_fd(ifindex, -1, 0);
if (err) {
fprintf(stderr, "Error when unsetting bpf prog_fd for ifindex(%d)\n", ifindex);
return err;
}
return 0;
cleanup:
close(msgsock);
close_sock:
close(sock);
unlink(SOCKET_NAME);
return err;
}
......@@ -24,10 +24,12 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/capability.h>
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/un.h>
#include <time.h>
#include <unistd.h>
......@@ -96,6 +98,7 @@ static bool opt_need_wakeup = true;
static u32 opt_num_xsks = 1;
static u32 prog_id;
static bool opt_busy_poll;
static bool opt_reduced_cap;
struct xsk_ring_stats {
unsigned long rx_npkts;
......@@ -154,6 +157,7 @@ struct xsk_socket_info {
static int num_socks;
struct xsk_socket_info *xsks[MAX_SOCKS];
int sock;
static unsigned long get_nsecs(void)
{
......@@ -461,6 +465,7 @@ static void *poller(void *arg)
static void remove_xdp_program(void)
{
u32 curr_prog_id = 0;
int cmd = CLOSE_CONN;
if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) {
printf("bpf_get_link_xdp_id failed\n");
......@@ -472,6 +477,13 @@ static void remove_xdp_program(void)
printf("couldn't find a prog id on a given interface\n");
else
printf("program on interface changed, not removing\n");
if (opt_reduced_cap) {
if (write(sock, &cmd, sizeof(int)) < 0) {
fprintf(stderr, "Error writing into stream socket: %s", strerror(errno));
exit(EXIT_FAILURE);
}
}
}
static void int_exit(int sig)
......@@ -854,7 +866,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
xsk->umem = umem;
cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
if (opt_num_xsks > 1)
if (opt_num_xsks > 1 || opt_reduced_cap)
cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
else
cfg.libbpf_flags = 0;
......@@ -913,6 +925,7 @@ static struct option long_options[] = {
{"app-stats", no_argument, 0, 'a'},
{"irq-string", no_argument, 0, 'I'},
{"busy-poll", no_argument, 0, 'B'},
{"reduce-cap", no_argument, 0, 'R'},
{0, 0, 0, 0}
};
......@@ -935,7 +948,7 @@ static void usage(const char *prog)
" -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n"
" -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n"
" -u, --unaligned Enable unaligned chunk placement\n"
" -M, --shared-umem Enable XDP_SHARED_UMEM\n"
" -M, --shared-umem Enable XDP_SHARED_UMEM (cannot be used with -R)\n"
" -F, --force Force loading the XDP prog\n"
" -d, --duration=n Duration in secs to run command.\n"
" Default: forever.\n"
......@@ -952,6 +965,7 @@ static void usage(const char *prog)
" -a, --app-stats Display application (syscall) statistics.\n"
" -I, --irq-string Display driver interrupt statistics for interface associated with irq-string.\n"
" -B, --busy-poll Busy poll.\n"
" -R, --reduce-cap Use reduced capabilities (cannot be used with -M)\n"
"\n";
fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE,
opt_batch_size, MIN_PKT_SIZE, MIN_PKT_SIZE,
......@@ -967,7 +981,7 @@ static void parse_command_line(int argc, char **argv)
opterr = 0;
for (;;) {
c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:s:P:xQaI:B",
c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:s:P:xQaI:BR",
long_options, &option_index);
if (c == -1)
break;
......@@ -1069,6 +1083,9 @@ static void parse_command_line(int argc, char **argv)
case 'B':
opt_busy_poll = 1;
break;
case 'R':
opt_reduced_cap = true;
break;
default:
usage(basename(argv[0]));
}
......@@ -1090,6 +1107,11 @@ static void parse_command_line(int argc, char **argv)
opt_xsk_frame_size);
usage(basename(argv[0]));
}
if (opt_reduced_cap && opt_num_xsks > 1) {
fprintf(stderr, "ERROR: -M and -R cannot be used together\n");
usage(basename(argv[0]));
}
}
static void kick_tx(struct xsk_socket_info *xsk)
......@@ -1487,19 +1509,116 @@ static void apply_setsockopt(struct xsk_socket_info *xsk)
exit_with_error(errno);
}
static int recv_xsks_map_fd_from_ctrl_node(int sock, int *_fd)
{
char cms[CMSG_SPACE(sizeof(int))];
struct cmsghdr *cmsg;
struct msghdr msg;
struct iovec iov;
int value;
int len;
iov.iov_base = &value;
iov.iov_len = sizeof(int);
msg.msg_name = 0;
msg.msg_namelen = 0;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
msg.msg_flags = 0;
msg.msg_control = (caddr_t)cms;
msg.msg_controllen = sizeof(cms);
len = recvmsg(sock, &msg, 0);
if (len < 0) {
fprintf(stderr, "Recvmsg failed length incorrect.\n");
return -EINVAL;
}
if (len == 0) {
fprintf(stderr, "Recvmsg failed no data\n");
return -EINVAL;
}
cmsg = CMSG_FIRSTHDR(&msg);
*_fd = *(int *)CMSG_DATA(cmsg);
return 0;
}
static int
recv_xsks_map_fd(int *xsks_map_fd)
{
struct sockaddr_un server;
int err;
sock = socket(AF_UNIX, SOCK_STREAM, 0);
if (sock < 0) {
fprintf(stderr, "Error opening socket stream: %s", strerror(errno));
return errno;
}
server.sun_family = AF_UNIX;
strcpy(server.sun_path, SOCKET_NAME);
if (connect(sock, (struct sockaddr *)&server, sizeof(struct sockaddr_un)) < 0) {
close(sock);
fprintf(stderr, "Error connecting stream socket: %s", strerror(errno));
return errno;
}
err = recv_xsks_map_fd_from_ctrl_node(sock, xsks_map_fd);
if (err) {
fprintf(stderr, "Error %d recieving fd\n", err);
return err;
}
return 0;
}
int main(int argc, char **argv)
{
struct __user_cap_header_struct hdr = { _LINUX_CAPABILITY_VERSION_3, 0 };
struct __user_cap_data_struct data[2] = { { 0 } };
struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
bool rx = false, tx = false;
struct xsk_umem_info *umem;
struct bpf_object *obj;
int xsks_map_fd = 0;
pthread_t pt;
int i, ret;
void *bufs;
parse_command_line(argc, argv);
if (opt_reduced_cap) {
if (capget(&hdr, data) < 0)
fprintf(stderr, "Error getting capabilities\n");
data->effective &= CAP_TO_MASK(CAP_NET_RAW);
data->permitted &= CAP_TO_MASK(CAP_NET_RAW);
if (capset(&hdr, data) < 0)
fprintf(stderr, "Setting capabilities failed\n");
if (capget(&hdr, data) < 0) {
fprintf(stderr, "Error getting capabilities\n");
} else {
fprintf(stderr, "Capabilities EFF %x Caps INH %x Caps Per %x\n",
data[0].effective, data[0].inheritable, data[0].permitted);
fprintf(stderr, "Capabilities EFF %x Caps INH %x Caps Per %x\n",
data[1].effective, data[1].inheritable, data[1].permitted);
}
} else {
if (setrlimit(RLIMIT_MEMLOCK, &r)) {
fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n",
strerror(errno));
exit(EXIT_FAILURE);
}
if (opt_num_xsks > 1)
load_xdp_program(argv, &obj);
}
/* Reserve memory for the umem. Use hugepages if unaligned chunk mode */
bufs = mmap(NULL, NUM_FRAMES * opt_xsk_frame_size,
......@@ -1534,6 +1653,21 @@ int main(int argc, char **argv)
if (opt_num_xsks > 1 && opt_bench != BENCH_TXONLY)
enter_xsks_into_map(obj);
if (opt_reduced_cap) {
ret = recv_xsks_map_fd(&xsks_map_fd);
if (ret) {
fprintf(stderr, "Error %d receiving xsks_map_fd\n", ret);
exit_with_error(ret);
}
if (xsks[0]->xsk) {
ret = xsk_socket__update_xskmap(xsks[0]->xsk, xsks_map_fd);
if (ret) {
fprintf(stderr, "Update of BPF map failed(%d)\n", ret);
exit_with_error(ret);
}
}
}
signal(SIGINT, int_exit);
signal(SIGTERM, int_exit);
signal(SIGABRT, int_exit);
......
......@@ -346,4 +346,6 @@ LIBBPF_0.3.0 {
btf__parse_split;
btf__new_empty_split;
btf__new_split;
xsk_setup_xdp_prog;
xsk_socket__update_xskmap;
} LIBBPF_0.2.0;
......@@ -566,8 +566,35 @@ static int xsk_set_bpf_maps(struct xsk_socket *xsk)
&xsk->fd, 0);
}
static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk)
{
char ifname[IFNAMSIZ];
struct xsk_ctx *ctx;
char *interface;
ctx = calloc(1, sizeof(*ctx));
if (!ctx)
return -ENOMEM;
interface = if_indextoname(ifindex, &ifname[0]);
if (!interface) {
free(ctx);
return -errno;
}
ctx->ifindex = ifindex;
strncpy(ctx->ifname, ifname, IFNAMSIZ - 1);
ctx->ifname[IFNAMSIZ - 1] = 0;
xsk->ctx = ctx;
return 0;
}
static int __xsk_setup_xdp_prog(struct xsk_socket *_xdp,
int *xsks_map_fd)
{
struct xsk_socket *xsk = _xdp;
struct xsk_ctx *ctx = xsk->ctx;
__u32 prog_id = 0;
int err;
......@@ -584,8 +611,7 @@ static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
err = xsk_load_xdp_prog(xsk);
if (err) {
xsk_delete_bpf_maps(xsk);
return err;
goto err_load_xdp_prog;
}
} else {
ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
......@@ -598,15 +624,29 @@ static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
}
}
if (xsk->rx)
if (xsk->rx) {
err = xsk_set_bpf_maps(xsk);
if (err) {
xsk_delete_bpf_maps(xsk);
if (!prog_id) {
goto err_set_bpf_maps;
} else {
close(ctx->prog_fd);
return err;
}
}
}
if (xsks_map_fd)
*xsks_map_fd = ctx->xsks_map_fd;
return 0;
err_set_bpf_maps:
close(ctx->prog_fd);
bpf_set_link_xdp_fd(ctx->ifindex, -1, 0);
err_load_xdp_prog:
xsk_delete_bpf_maps(xsk);
return err;
}
static struct xsk_ctx *xsk_get_ctx(struct xsk_umem *umem, int ifindex,
......@@ -689,6 +729,40 @@ static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk,
return ctx;
}
static void xsk_destroy_xsk_struct(struct xsk_socket *xsk)
{
free(xsk->ctx);
free(xsk);
}
int xsk_socket__update_xskmap(struct xsk_socket *xsk, int fd)
{
xsk->ctx->xsks_map_fd = fd;
return xsk_set_bpf_maps(xsk);
}
int xsk_setup_xdp_prog(int ifindex, int *xsks_map_fd)
{
struct xsk_socket *xsk;
int res;
xsk = calloc(1, sizeof(*xsk));
if (!xsk)
return -ENOMEM;
res = xsk_create_xsk_struct(ifindex, xsk);
if (res) {
free(xsk);
return -EINVAL;
}
res = __xsk_setup_xdp_prog(xsk, xsks_map_fd);
xsk_destroy_xsk_struct(xsk);
return res;
}
int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
const char *ifname,
__u32 queue_id, struct xsk_umem *umem,
......@@ -838,7 +912,7 @@ int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
ctx->prog_fd = -1;
if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
err = xsk_setup_xdp_prog(xsk);
err = __xsk_setup_xdp_prog(xsk, NULL);
if (err)
goto out_mmap_tx;
}
......
......@@ -204,6 +204,11 @@ struct xsk_umem_config {
__u32 flags;
};
LIBBPF_API int xsk_setup_xdp_prog(int ifindex,
int *xsks_map_fd);
LIBBPF_API int xsk_socket__update_xskmap(struct xsk_socket *xsk,
int xsks_map_fd);
/* Flags for the libbpf_flags field. */
#define XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD (1 << 0)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment