helpers.h 15.5 KB
Newer Older
1
R"********(
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * Copyright (c) 2015 PLUMgrid, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
Brenden Blanco's avatar
Brenden Blanco committed
17 18 19
#ifndef __BPF_HELPERS_H
#define __BPF_HELPERS_H

20
#include <uapi/linux/bpf.h>
21
#include <uapi/linux/if_packet.h>
Brenden Blanco's avatar
Brenden Blanco committed
22 23
#include <linux/version.h>

24 25 26 27
#ifndef CONFIG_BPF_SYSCALL
#error "CONFIG_BPF_SYSCALL is undefined, please check your .config or ask your Linux distro to enable this feature"
#endif

28 29 30 31 32 33
#ifdef PERF_MAX_STACK_DEPTH
#define BPF_MAX_STACK_DEPTH PERF_MAX_STACK_DEPTH
#else
#define BPF_MAX_STACK_DEPTH 127
#endif

Brenden Blanco's avatar
Brenden Blanco committed
34 35 36 37 38 39
/* helper macro to place programs, maps, license in
 * different sections in elf_bpf file. Section names
 * are interpreted by elf_bpf loader
 */
#define SEC(NAME) __attribute__((section(NAME), used))

Brenden Blanco's avatar
Brenden Blanco committed
40 41 42 43 44
// Changes to the macro require changes in BFrontendAction classes
#define BPF_TABLE(_table_type, _key_type, _leaf_type, _name, _max_entries) \
struct _name##_table_t { \
  _key_type key; \
  _leaf_type leaf; \
45 46 47
  _leaf_type * (*lookup) (_key_type *); \
  _leaf_type * (*lookup_or_init) (_key_type *, _leaf_type *); \
  int (*update) (_key_type *, _leaf_type *); \
Brenden Blanco's avatar
Brenden Blanco committed
48
  int (*delete) (_key_type *); \
49
  void (*call) (void *, int index); \
50
  void (*increment) (_key_type); \
51
  int (*get_stackid) (void *, u64); \
Brenden Blanco's avatar
Brenden Blanco committed
52 53 54 55 56
  _leaf_type data[_max_entries]; \
}; \
__attribute__((section("maps/" _table_type))) \
struct _name##_table_t _name

57 58 59
// define a table same as above but allow it to be referenced by other modules
#define BPF_TABLE_PUBLIC(_table_type, _key_type, _leaf_type, _name, _max_entries) \
BPF_TABLE(_table_type, _key_type, _leaf_type, _name, _max_entries); \
60 61 62
__attribute__((section("maps/export"))) \
struct _name##_table_t __##_name

63 64 65 66 67 68 69 70 71 72 73 74 75
// Table for pushing custom events to userspace via ring buffer
#define BPF_PERF_OUTPUT(_name) \
struct _name##_table_t { \
  int key; \
  u32 leaf; \
  /* map.perf_submit(ctx, data, data_size) */ \
  int (*perf_submit) (void *, void *, u32); \
  u32 data[0]; \
}; \
__attribute__((section("maps/perf_output"))) \
struct _name##_table_t _name

// Table for reading hw perf cpu counters
76 77 78 79 80 81 82 83 84 85 86
#define BPF_PERF_ARRAY(_name, _max_entries) \
struct _name##_table_t { \
  int key; \
  u32 leaf; \
  /* counter = map.perf_read(index) */ \
  u64 (*perf_read) (int); \
  u32 data[_max_entries]; \
}; \
__attribute__((section("maps/perf_array"))) \
struct _name##_table_t _name

87 88 89 90 91 92 93 94 95 96 97 98 99 100
#define BPF_HASH1(_name) \
  BPF_TABLE("hash", u64, u64, _name, 10240)
#define BPF_HASH2(_name, _key_type) \
  BPF_TABLE("hash", _key_type, u64, _name, 10240)
#define BPF_HASH3(_name, _key_type, _leaf_type) \
  BPF_TABLE("hash", _key_type, _leaf_type, _name, 10240)
// helper for default-variable macro function
#define BPF_HASHX(_1, _2, _3, NAME, ...) NAME

// Define a hash function, some arguments optional
// BPF_HASH(name, key_type=u64, leaf_type=u64, size=10240)
#define BPF_HASH(...) \
  BPF_HASHX(__VA_ARGS__, BPF_HASH3, BPF_HASH2, BPF_HASH1)(__VA_ARGS__)

101 102 103 104 105 106 107 108 109 110 111 112 113
#define BPF_HIST1(_name) \
  BPF_TABLE("histogram", int, u64, _name, 64)
#define BPF_HIST2(_name, _key_type) \
  BPF_TABLE("histogram", _key_type, u64, _name, 64)
#define BPF_HIST3(_name, _key_type, _size) \
  BPF_TABLE("histogram", _key_type, u64, _name, _size)
#define BPF_HISTX(_1, _2, _3, NAME, ...) NAME

// Define a histogram, some arguments optional
// BPF_HISTOGRAM(name, key_type=int, size=64)
#define BPF_HISTOGRAM(...) \
  BPF_HISTX(__VA_ARGS__, BPF_HIST3, BPF_HIST2, BPF_HIST1)(__VA_ARGS__)

114 115 116 117
struct bpf_stacktrace {
  u64 ip[BPF_MAX_STACK_DEPTH];
};

118
#define BPF_STACK_TRACE(_name, _max_entries) \
119
  BPF_TABLE("stacktrace", int, struct bpf_stacktrace, _name, _max_entries);
120

121
// packet parsing state machine helpers
122 123
#define cursor_advance(_cursor, _len) \
  ({ void *_tmp = _cursor; _cursor += _len; _tmp; })
124

Brenden Blanco's avatar
Brenden Blanco committed
125 126 127 128
char _license[4] SEC("license") = "GPL";

unsigned _version SEC("version") = LINUX_VERSION_CODE;

Brenden Blanco's avatar
Brenden Blanco committed
129 130
/* helper functions called from eBPF programs written in C */
static void *(*bpf_map_lookup_elem)(void *map, void *key) =
131
  (void *) BPF_FUNC_map_lookup_elem;
132
static int (*bpf_map_update_elem)(void *map, void *key, void *value, u64 flags) =
133
  (void *) BPF_FUNC_map_update_elem;
Brenden Blanco's avatar
Brenden Blanco committed
134
static int (*bpf_map_delete_elem)(void *map, void *key) =
135
  (void *) BPF_FUNC_map_delete_elem;
136
static int (*bpf_probe_read)(void *dst, u64 size, void *unsafe_ptr) =
137
  (void *) BPF_FUNC_probe_read;
138
static u64 (*bpf_ktime_get_ns)(void) =
139
  (void *) BPF_FUNC_ktime_get_ns;
140
static u32 (*bpf_get_prandom_u32)(void) =
141
  (void *) BPF_FUNC_get_prandom_u32;
142
static int (*bpf_trace_printk_)(const char *fmt, u64 fmt_size, ...) =
143
  (void *) BPF_FUNC_trace_printk;
144
int bpf_trace_printk(const char *fmt, ...) asm("llvm.bpf.extra");
145 146 147
static void bpf_tail_call_(u64 map_fd, void *ctx, int index) {
  ((void (*)(void *, u64, int))BPF_FUNC_tail_call)(ctx, map_fd, index);
}
148 149
static int (*bpf_clone_redirect)(void *ctx, int ifindex, u32 flags) =
  (void *) BPF_FUNC_clone_redirect;
150
static u64 (*bpf_get_smp_processor_id)(void) =
151
  (void *) BPF_FUNC_get_smp_processor_id;
152
static u64 (*bpf_get_current_pid_tgid)(void) =
153
  (void *) BPF_FUNC_get_current_pid_tgid;
154
static u64 (*bpf_get_current_uid_gid)(void) =
155
  (void *) BPF_FUNC_get_current_uid_gid;
156
static int (*bpf_get_current_comm)(void *buf, int buf_size) =
157
  (void *) BPF_FUNC_get_current_comm;
158
static u64 (*bpf_get_cgroup_classid)(void *ctx) =
159
  (void *) BPF_FUNC_get_cgroup_classid;
160
static u64 (*bpf_skb_vlan_push)(void *ctx, u16 proto, u16 vlan_tci) =
161
  (void *) BPF_FUNC_skb_vlan_push;
162
static u64 (*bpf_skb_vlan_pop)(void *ctx) =
163
  (void *) BPF_FUNC_skb_vlan_pop;
164
static int (*bpf_skb_get_tunnel_key)(void *ctx, void *to, u32 size, u64 flags) =
165
  (void *) BPF_FUNC_skb_get_tunnel_key;
166
static int (*bpf_skb_set_tunnel_key)(void *ctx, void *from, u32 size, u64 flags) =
167
  (void *) BPF_FUNC_skb_set_tunnel_key;
168 169 170 171 172 173 174 175
static int (*bpf_perf_event_read)(void *map, u32 index) =
  (void *) BPF_FUNC_perf_event_read;
static int (*bpf_redirect)(int ifindex, u32 flags) =
  (void *) BPF_FUNC_redirect;
static u32 (*bpf_get_route_realm)(void *ctx) =
  (void *) BPF_FUNC_get_route_realm;
static int (*bpf_perf_event_output)(void *ctx, void *map, u32 index, void *data, u32 size) =
  (void *) BPF_FUNC_perf_event_output;
176 177
static int (*bpf_skb_load_bytes)(void *ctx, int offset, void *to, u32 len) =
  (void *) BPF_FUNC_skb_load_bytes;
178
static int (*bpf_get_stackid_)(void *ctx, void *map, u64 flags) =
179
  (void *) BPF_FUNC_get_stackid;
180 181 182 183
static inline __attribute__((always_inline))
int bpf_get_stackid(uintptr_t map, void *ctx, u64 flags) {
  return bpf_get_stackid_(ctx, (void *)map, flags);
}
184 185
static int (*bpf_csum_diff)(void *from, u64 from_size, void *to, u64 to_size, u64 seed) =
  (void *) BPF_FUNC_csum_diff;
Brenden Blanco's avatar
Brenden Blanco committed
186 187 188 189 190 191

/* llvm builtin functions that eBPF C program may use to
 * emit BPF_LD_ABS and BPF_LD_IND instructions
 */
struct sk_buff;
unsigned long long load_byte(void *skb,
192
  unsigned long long off) asm("llvm.bpf.load.byte");
Brenden Blanco's avatar
Brenden Blanco committed
193
unsigned long long load_half(void *skb,
194
  unsigned long long off) asm("llvm.bpf.load.half");
Brenden Blanco's avatar
Brenden Blanco committed
195
unsigned long long load_word(void *skb,
196
  unsigned long long off) asm("llvm.bpf.load.word");
Brenden Blanco's avatar
Brenden Blanco committed
197 198 199 200 201

/* a helper structure used by eBPF C program
 * to describe map attributes to elf_bpf loader
 */
struct bpf_map_def {
202 203 204 205
  unsigned int type;
  unsigned int key_size;
  unsigned int value_size;
  unsigned int max_entries;
Brenden Blanco's avatar
Brenden Blanco committed
206 207 208
};

static int (*bpf_skb_store_bytes)(void *ctx, unsigned long long off, void *from,
209 210
                                  unsigned long long len, unsigned long long flags) =
  (void *) BPF_FUNC_skb_store_bytes;
Brenden Blanco's avatar
Brenden Blanco committed
211
static int (*bpf_l3_csum_replace)(void *ctx, unsigned long long off, unsigned long long from,
212 213
                                  unsigned long long to, unsigned long long flags) =
  (void *) BPF_FUNC_l3_csum_replace;
Brenden Blanco's avatar
Brenden Blanco committed
214
static int (*bpf_l4_csum_replace)(void *ctx, unsigned long long off, unsigned long long from,
215 216
                                  unsigned long long to, unsigned long long flags) =
  (void *) BPF_FUNC_l4_csum_replace;
Brenden Blanco's avatar
Brenden Blanco committed
217

Brenden Blanco's avatar
Brenden Blanco committed
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251
static inline u16 bpf_ntohs(u16 val) {
  /* will be recognized by gcc into rotate insn and eventually rolw 8 */
  return (val << 8) | (val >> 8);
}

static inline u32 bpf_ntohl(u32 val) {
  /* gcc will use bswapsi2 insn */
  return __builtin_bswap32(val);
}

static inline u64 bpf_ntohll(u64 val) {
  /* gcc will use bswapdi2 insn */
  return __builtin_bswap64(val);
}

static inline unsigned __int128 bpf_ntoh128(unsigned __int128 val) {
  return (((unsigned __int128)bpf_ntohll(val) << 64) | (u64)bpf_ntohll(val >> 64));
}

static inline u16 bpf_htons(u16 val) {
  return bpf_ntohs(val);
}

static inline u32 bpf_htonl(u32 val) {
  return bpf_ntohl(val);
}
static inline u64 bpf_htonll(u64 val) {
  return bpf_ntohll(val);
}
static inline unsigned __int128 bpf_hton128(unsigned __int128 val) {
  return bpf_ntoh128(val);
}

static inline u64 load_dword(void *skb, u64 off) {
252
  return ((u64)load_word(skb, off) << 32) | load_word(skb, off + 4);
Brenden Blanco's avatar
Brenden Blanco committed
253 254 255 256 257 258 259 260 261 262 263 264 265 266
}

void bpf_store_byte(void *skb, u64 off, u64 val) asm("llvm.bpf.store.byte");
void bpf_store_half(void *skb, u64 off, u64 val) asm("llvm.bpf.store.half");
void bpf_store_word(void *skb, u64 off, u64 val) asm("llvm.bpf.store.word");
u64 bpf_pseudo_fd(u64, u64) asm("llvm.bpf.pseudo");
static inline void bpf_store_dword(void *skb, u64 off, u64 val) {
  bpf_store_word(skb, off, (u32)val);
  bpf_store_word(skb, off + 4, val >> 32);
}

#define MASK(_n) ((_n) < 64 ? (1ull << (_n)) - 1 : ((u64)-1LL))
#define MASK128(_n) ((_n) < 128 ? ((unsigned __int128)1 << (_n)) - 1 : ((unsigned __int128)-1))

Brendan Gregg's avatar
Brendan Gregg committed
267 268
static unsigned int bpf_log2(unsigned int v)
{
269 270 271 272 273 274 275 276 277
  unsigned int r;
  unsigned int shift;

  r = (v > 0xFFFF) << 4; v >>= r;
  shift = (v > 0xFF) << 3; v >>= shift; r |= shift;
  shift = (v > 0xF) << 2; v >>= shift; r |= shift;
  shift = (v > 0x3) << 1; v >>= shift; r |= shift;
  r |= (v >> 1);
  return r;
Brendan Gregg's avatar
Brendan Gregg committed
278 279 280 281
}

static unsigned int bpf_log2l(unsigned long v)
{
282 283 284 285 286
  unsigned int hi = v >> 32;
  if (hi)
    return bpf_log2(hi) + 32 + 1;
  else
    return bpf_log2(v) + 1;
Brendan Gregg's avatar
Brendan Gregg committed
287 288
}

Brenden Blanco's avatar
Brenden Blanco committed
289 290
struct bpf_context;

291
static inline __attribute__((always_inline))
Brenden Blanco's avatar
Brenden Blanco committed
292 293 294 295 296 297 298 299 300 301 302 303 304 305
SEC("helpers")
u64 bpf_dext_pkt(void *pkt, u64 off, u64 bofs, u64 bsz) {
  if (bofs == 0 && bsz == 8) {
    return load_byte(pkt, off);
  } else if (bofs + bsz <= 8) {
    return load_byte(pkt, off) >> (8 - (bofs + bsz))  &  MASK(bsz);
  } else if (bofs == 0 && bsz == 16) {
    return load_half(pkt, off);
  } else if (bofs + bsz <= 16) {
    return load_half(pkt, off) >> (16 - (bofs + bsz))  &  MASK(bsz);
  } else if (bofs == 0 && bsz == 32) {
    return load_word(pkt, off);
  } else if (bofs + bsz <= 32) {
    return load_word(pkt, off) >> (32 - (bofs + bsz))  &  MASK(bsz);
306 307
  } else if (bofs == 0 && bsz == 64) {
    return load_dword(pkt, off);
Brenden Blanco's avatar
Brenden Blanco committed
308
  } else if (bofs + bsz <= 64) {
309
    return load_dword(pkt, off) >> (64 - (bofs + bsz))  &  MASK(bsz);
Brenden Blanco's avatar
Brenden Blanco committed
310 311 312 313
  }
  return 0;
}

314
static inline __attribute__((always_inline))
Brenden Blanco's avatar
Brenden Blanco committed
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
SEC("helpers")
void bpf_dins_pkt(void *pkt, u64 off, u64 bofs, u64 bsz, u64 val) {
  // The load_xxx function does a bswap before returning the short/word/dword,
  // so the value in register will always be host endian. However, the bytes
  // written back need to be in network order.
  if (bofs == 0 && bsz == 8) {
    bpf_skb_store_bytes(pkt, off, &val, 1, 0);
  } else if (bofs + bsz <= 8) {
    u8 v = load_byte(pkt, off);
    v &= ~(MASK(bsz) << (8 - (bofs + bsz)));
    v |= ((val & MASK(bsz)) << (8 - (bofs + bsz)));
    bpf_skb_store_bytes(pkt, off, &v, 1, 0);
  } else if (bofs == 0 && bsz == 16) {
    u16 v = bpf_htons(val);
    bpf_skb_store_bytes(pkt, off, &v, 2, 0);
  } else if (bofs + bsz <= 16) {
    u16 v = load_half(pkt, off);
    v &= ~(MASK(bsz) << (16 - (bofs + bsz)));
    v |= ((val & MASK(bsz)) << (16 - (bofs + bsz)));
    v = bpf_htons(v);
    bpf_skb_store_bytes(pkt, off, &v, 2, 0);
  } else if (bofs == 0 && bsz == 32) {
    u32 v = bpf_htonl(val);
    bpf_skb_store_bytes(pkt, off, &v, 4, 0);
  } else if (bofs + bsz <= 32) {
    u32 v = load_word(pkt, off);
    v &= ~(MASK(bsz) << (32 - (bofs + bsz)));
    v |= ((val & MASK(bsz)) << (32 - (bofs + bsz)));
    v = bpf_htonl(v);
    bpf_skb_store_bytes(pkt, off, &v, 4, 0);
  } else if (bofs == 0 && bsz == 64) {
    u64 v = bpf_htonll(val);
    bpf_skb_store_bytes(pkt, off, &v, 8, 0);
  } else if (bofs + bsz <= 64) {
    u64 v = load_dword(pkt, off);
    v &= ~(MASK(bsz) << (64 - (bofs + bsz)));
    v |= ((val & MASK(bsz)) << (64 - (bofs + bsz)));
    v = bpf_htonll(v);
    bpf_skb_store_bytes(pkt, off, &v, 8, 0);
  }
}

357
static inline __attribute__((always_inline))
Brenden Blanco's avatar
Brenden Blanco committed
358 359 360 361 362
SEC("helpers")
void * bpf_map_lookup_elem_(uintptr_t map, void *key) {
  return bpf_map_lookup_elem((void *)map, key);
}

363
static inline __attribute__((always_inline))
Brenden Blanco's avatar
Brenden Blanco committed
364 365 366 367 368
SEC("helpers")
int bpf_map_update_elem_(uintptr_t map, void *key, void *value, u64 flags) {
  return bpf_map_update_elem((void *)map, key, value, flags);
}

369
static inline __attribute__((always_inline))
Brenden Blanco's avatar
Brenden Blanco committed
370 371 372 373 374
SEC("helpers")
int bpf_map_delete_elem_(uintptr_t map, void *key) {
  return bpf_map_delete_elem((void *)map, key);
}

375
static inline __attribute__((always_inline))
Brenden Blanco's avatar
Brenden Blanco committed
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
SEC("helpers")
int bpf_l3_csum_replace_(void *ctx, u64 off, u64 from, u64 to, u64 flags) {
  switch (flags & 0xf) {
    case 2:
      return bpf_l3_csum_replace(ctx, off, bpf_htons(from), bpf_htons(to), flags);
    case 4:
      return bpf_l3_csum_replace(ctx, off, bpf_htonl(from), bpf_htonl(to), flags);
    case 8:
      return bpf_l3_csum_replace(ctx, off, bpf_htonll(from), bpf_htonll(to), flags);
    default:
      {}
  }
  return bpf_l3_csum_replace(ctx, off, from, to, flags);
}

391
static inline __attribute__((always_inline))
Brenden Blanco's avatar
Brenden Blanco committed
392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
SEC("helpers")
int bpf_l4_csum_replace_(void *ctx, u64 off, u64 from, u64 to, u64 flags) {
  switch (flags & 0xf) {
    case 2:
      return bpf_l4_csum_replace(ctx, off, bpf_htons(from), bpf_htons(to), flags);
    case 4:
      return bpf_l4_csum_replace(ctx, off, bpf_htonl(from), bpf_htonl(to), flags);
    case 8:
      return bpf_l4_csum_replace(ctx, off, bpf_htonll(from), bpf_htonll(to), flags);
    default:
      {}
  }
  return bpf_l4_csum_replace(ctx, off, from, to, flags);
}

407 408
int incr_cksum_l3(void *off, u64 oldval, u64 newval) asm("llvm.bpf.extra");
int incr_cksum_l4(void *off, u64 oldval, u64 newval, u64 flags) asm("llvm.bpf.extra");
409
int bpf_num_cpus() asm("llvm.bpf.extra");
410

411 412
#define lock_xadd(ptr, val) ((void)__sync_fetch_and_add(ptr, val))

413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436
#ifdef __powerpc__
#define PT_REGS_PARM1(ctx)	((ctx)->gpr[3])
#define PT_REGS_PARM2(ctx)	((ctx)->gpr[4])
#define PT_REGS_PARM3(ctx)	((ctx)->gpr[5])
#define PT_REGS_PARM4(ctx)	((ctx)->gpr[6])
#define PT_REGS_PARM5(ctx)	((ctx)->gpr[7])
#define PT_REGS_PARM6(ctx)	((ctx)->gpr[8])
#define PT_REGS_RC(ctx)		((ctx)->gpr[3])
#define PT_REGS_IP(ctx)		((ctx)->nip)
#define PT_REGS_SP(ctx)		((ctx)->sp)
#elif defined(__x86_64__)
#define PT_REGS_PARM1(ctx)	((ctx)->di)
#define PT_REGS_PARM2(ctx)	((ctx)->si)
#define PT_REGS_PARM3(ctx)	((ctx)->dx)
#define PT_REGS_PARM4(ctx)	((ctx)->cx)
#define PT_REGS_PARM5(ctx)	((ctx)->r8)
#define PT_REGS_PARM6(ctx)	((ctx)->r9)
#define PT_REGS_RC(ctx)		((ctx)->ax)
#define PT_REGS_IP(ctx)		((ctx)->ip)
#define PT_REGS_SP(ctx)		((ctx)->sp)
#else
#error "bcc does not support this platform yet"
#endif

437 438 439 440 441
#define bpf_usdt_readarg(probearg, ctx) _bpf_readarg_##probearg(ctx)
#define bpf_usdt_readarg_p(probearg, ctx, buf, len) {\
  u64 __addr = bpf_usdt_readarg(probearg, ctx); \
  bpf_probe_read(buf, len, (void *)__addr); }

Brenden Blanco's avatar
Brenden Blanco committed
442
#endif
443
)********"