Merge pull request #1524 from iovisor/yhs_dev

usdt: better handling for probes existing in multiple binary files

src/cc/api/BPF.cc

@@ -616,7 +616,7 @@ StatusTuple USDT::init() {
   ::USDT::Context ctx(binary_path_);
   if (!ctx.loaded())
     return StatusTuple(-1, "Unable to load USDT " + print_name());
-  auto probe = ctx.get(name_);
+  auto probe = ctx.get(provider_, name_);
   if (probe == nullptr)
     return StatusTuple(-1, "Unable to find USDT " + print_name());
 

src/cc/bcc_usdt.h

@@ -22,7 +22,7 @@ extern "C" {
 
 #include <stdint.h>
 
-void *bcc_usdt_new_frompid(int pid);
+void *bcc_usdt_new_frompid(int pid, const char *path);
 void *bcc_usdt_new_frompath(const char *path);
 void bcc_usdt_close(void *usdt);
 
@@ -37,6 +37,7 @@ struct bcc_usdt {
 
 struct bcc_usdt_location {
     uint64_t address;
+    const char *bin_path;
 };
 
 #define BCC_USDT_ARGUMENT_NONE                0x0
@@ -60,9 +61,11 @@ struct bcc_usdt_argument {
 
 typedef void (*bcc_usdt_cb)(struct bcc_usdt *);
 void bcc_usdt_foreach(void *usdt, bcc_usdt_cb callback);
-int bcc_usdt_get_location(void *usdt, const char *probe_name,
+int bcc_usdt_get_location(void *usdt, const char *provider_name,
+                          const char *probe_name,
                           int index, struct bcc_usdt_location *location);
-int bcc_usdt_get_argument(void *usdt, const char *probe_name,
+int bcc_usdt_get_argument(void *usdt, const char *provider_name,
+                          const char *probe_name,
                           int location_index, int argument_index,
                           struct bcc_usdt_argument *argument);
 

src/cc/link_all.cc

@@ -13,7 +13,7 @@ namespace {
       if (::getenv("bar") != (char *)-1)
         return;
 
-      (void)bcc_usdt_new_frompid(-1);
+      (void)bcc_usdt_new_frompid(-1, nullptr);
       (void)bcc_usdt_new_frompath(nullptr);
       (void)bcc_usdt_close(nullptr);
     }

src/cc/usdt.h

@@ -170,12 +170,13 @@ public:
 
 struct Location {
   uint64_t address_;
+  std::string bin_path_;
   std::vector<Argument> arguments_;
-  Location(uint64_t addr, const char *arg_fmt);
+  Location(uint64_t addr, const std::string &bin_path, const char *arg_fmt);
 };
 
 class Probe {
-  std::string bin_path_;
+  std::string bin_path_; // initial bin_path when Probe is created
   std::string provider_;
   std::string name_;
   uint64_t semaphore_;
@@ -184,7 +185,7 @@ class Probe {
 
   optional<int> pid_;
   ProcMountNS *mount_ns_;
-  optional<bool> in_shared_object_;
+  std::unordered_map<std::string, bool> object_type_map_; // bin_path => is shared lib?
 
   optional<std::string> attached_to_;
   optional<uint64_t> attached_semaphore_;
@@ -192,9 +193,10 @@ class Probe {
   std::string largest_arg_type(size_t arg_n);
 
   bool add_to_semaphore(int16_t val);
-  bool resolve_global_address(uint64_t *global, const uint64_t addr);
+  bool resolve_global_address(uint64_t *global, const std::string &bin_path,
+                              const uint64_t addr);
   bool lookup_semaphore_addr(uint64_t *address);
-  void add_location(uint64_t addr, const char *fmt);
+  void add_location(uint64_t addr, const std::string &bin_path, const char *fmt);
 
 public:
   Probe(const char *bin_path, const char *provider, const char *name,
@@ -205,6 +207,7 @@ public:
   uint64_t semaphore()   const { return semaphore_; }
 
   uint64_t address(size_t n = 0) const { return locations_[n].address_; }
+  const char *location_bin_path(size_t n = 0) const { return locations_[n].bin_path_.c_str(); }
   const Location &location(size_t n) const { return locations_[n]; }
   bool usdt_getarg(std::ostream &stream);
   std::string get_arg_ctype(int arg_index) {
@@ -217,7 +220,7 @@ public:
   bool disable();
   bool enabled() const { return !!attached_to_; }
 
-  bool in_shared_object();
+  bool in_shared_object(const std::string &bin_path);
   const std::string &name() { return name_; }
   const std::string &bin_path() { return bin_path_; }
   const std::string &provider() { return provider_; }
@@ -246,6 +249,7 @@ class Context {
 public:
   Context(const std::string &bin_path);
   Context(int pid);
+  Context(int pid, const std::string &bin_path);
   ~Context();
 
   optional<int> pid() const { return pid_; }
@@ -255,6 +259,7 @@ public:
   ino_t inode() const { return mount_ns_instance_->target_ino(); }
 
   Probe *get(const std::string &probe_name);
+  Probe *get(const std::string &provider_name, const std::string &probe_name);
   Probe *get(int pos) { return probes_[pos].get(); }
 
   bool enable_probe(const std::string &probe_name, const std::string &fn_name);

src/cc/usdt/usdt.cc

@@ -31,7 +31,10 @@
 
 namespace USDT {
 
-Location::Location(uint64_t addr, const char *arg_fmt) : address_(addr) {
+Location::Location(uint64_t addr, const std::string &bin_path, const char *arg_fmt)
+    : address_(addr),
+      bin_path_(bin_path) {
+
 #ifdef __aarch64__
   ArgumentParser_aarch64 parser(arg_fmt);
 #elif __powerpc64__
@@ -56,18 +59,19 @@ Probe::Probe(const char *bin_path, const char *provider, const char *name,
       pid_(pid),
       mount_ns_(ns) {}
 
-bool Probe::in_shared_object() {
-  if (!in_shared_object_) {
-    ProcMountNSGuard g(mount_ns_);
-    in_shared_object_ = bcc_elf_is_shared_obj(bin_path_.c_str());
-  }
-  return in_shared_object_.value();
+bool Probe::in_shared_object(const std::string &bin_path) {
+    if (object_type_map_.find(bin_path) == object_type_map_.end()) {
+      ProcMountNSGuard g(mount_ns_);
+      return (object_type_map_[bin_path] = bcc_elf_is_shared_obj(bin_path.c_str()));
+    }
+    return object_type_map_[bin_path];
 }
 
-bool Probe::resolve_global_address(uint64_t *global, const uint64_t addr) {
-  if (in_shared_object()) {
+bool Probe::resolve_global_address(uint64_t *global, const std::string &bin_path,
+                                   const uint64_t addr) {
+  if (in_shared_object(bin_path)) {
     return (pid_ &&
-            !bcc_resolve_global_addr(*pid_, bin_path_.c_str(), addr, global));
+            !bcc_resolve_global_addr(*pid_, bin_path.c_str(), addr, global));
   }
 
   *global = addr;
@@ -79,7 +83,7 @@ bool Probe::add_to_semaphore(int16_t val) {
 
   if (!attached_semaphore_) {
     uint64_t addr;
-    if (!resolve_global_address(&addr, semaphore_))
+    if (!resolve_global_address(&addr, bin_path_, semaphore_))
       return false;
     attached_semaphore_ = addr;
   }
@@ -175,7 +179,7 @@ bool Probe::usdt_getarg(std::ostream &stream) {
     if (locations_.size() == 1) {
       Location &location = locations_.front();
       stream << "  ";
-      if (!location.arguments_[arg_n].assign_to_local(stream, cptr, bin_path_,
+      if (!location.arguments_[arg_n].assign_to_local(stream, cptr, location.bin_path_,
                                                       pid_))
         return false;
       stream << "\n  return 0;\n}\n";
@@ -184,11 +188,12 @@ bool Probe::usdt_getarg(std::ostream &stream) {
       for (Location &location : locations_) {
         uint64_t global_address;
 
-        if (!resolve_global_address(&global_address, location.address_))
+        if (!resolve_global_address(&global_address, location.bin_path_,
+                                    location.address_))
           return false;
 
         tfm::format(stream, "  case 0x%xULL: ", global_address);
-        if (!location.arguments_[arg_n].assign_to_local(stream, cptr, bin_path_,
+        if (!location.arguments_[arg_n].assign_to_local(stream, cptr, location.bin_path_,
                                                         pid_))
           return false;
 
@@ -201,18 +206,18 @@ bool Probe::usdt_getarg(std::ostream &stream) {
   return true;
 }
 
-void Probe::add_location(uint64_t addr, const char *fmt) {
-  locations_.emplace_back(addr, fmt);
+void Probe::add_location(uint64_t addr, const std::string &bin_path, const char *fmt) {
+  locations_.emplace_back(addr, bin_path, fmt);
 }
 
 void Probe::finalize_locations() {
   std::sort(locations_.begin(), locations_.end(),
             [](const Location &a, const Location &b) {
-              return a.address_ < b.address_;
+              return a.bin_path_ < b.bin_path_ || a.address_ < b.address_;
             });
   auto last = std::unique(locations_.begin(), locations_.end(),
                           [](const Location &a, const Location &b) {
-                            return a.address_ == b.address_;
+                            return a.bin_path_ == b.bin_path_ && a.address_ == b.address_;
                           });
   locations_.erase(last, locations_.end());
 }
@@ -239,7 +244,7 @@ int Context::_each_module(const char *modpath, uint64_t, uint64_t, uint64_t,
 void Context::add_probe(const char *binpath, const struct bcc_elf_usdt *probe) {
   for (auto &p : probes_) {
     if (p->provider_ == probe->provider && p->name_ == probe->name) {
-      p->add_location(probe->pc, probe->arg_fmt);
+      p->add_location(probe->pc, binpath, probe->arg_fmt);
       return;
     }
   }
@@ -247,7 +252,7 @@ void Context::add_probe(const char *binpath, const struct bcc_elf_usdt *probe) {
   probes_.emplace_back(
       new Probe(binpath, probe->provider, probe->name, probe->semaphore, pid_,
 	mount_ns_instance_.get()));
-  probes_.back()->add_location(probe->pc, probe->arg_fmt);
+  probes_.back()->add_location(probe->pc, binpath, probe->arg_fmt);
 }
 
 std::string Context::resolve_bin_path(const std::string &bin_path) {
@@ -272,13 +277,41 @@ Probe *Context::get(const std::string &probe_name) {
   return nullptr;
 }
 
+Probe *Context::get(const std::string &provider_name,
+                    const std::string &probe_name) {
+  for (auto &p : probes_) {
+    if (p->provider_ == provider_name && p->name_ == probe_name)
+      return p.get();
+  }
+  return nullptr;
+}
+
 bool Context::enable_probe(const std::string &probe_name,
                            const std::string &fn_name) {
   if (pid_stat_ && pid_stat_->is_stale())
     return false;
 
-  auto p = get(probe_name);
-  return p && p->enable(fn_name);
+  // FIXME: we may have issues here if the context has two same probes's
+  // but different providers. For example, libc:setjmp and rtld:setjmp,
+  // libc:lll_futex_wait and rtld:lll_futex_wait.
+  Probe *found_probe = nullptr;
+  for (auto &p : probes_) {
+    if (p->name_ == probe_name) {
+      if (found_probe != nullptr) {
+         fprintf(stderr, "Two same-name probes (%s) but different providers\n",
+                 probe_name.c_str());
+         return false;
+      }
+      found_probe = p.get();
+    }
+  }
+
+  if (found_probe != nullptr) {
+    found_probe->enable(fn_name);
+    return true;
+  }
+
+  return false;
 }
 
 void Context::each(each_cb callback) {
@@ -300,7 +333,7 @@ void Context::each_uprobe(each_uprobe_cb callback) {
       continue;
 
     for (Location &loc : p->locations_) {
-      callback(p->bin_path_.c_str(), p->attached_to_->c_str(), loc.address_,
+      callback(loc.bin_path_.c_str(), p->attached_to_->c_str(), loc.address_,
                pid_.value_or(-1));
     }
   }
@@ -332,6 +365,22 @@ Context::Context(int pid) : pid_(pid), pid_stat_(pid),
     probe->finalize_locations();
 }
 
+Context::Context(int pid, const std::string &bin_path)
+    : pid_(pid), pid_stat_(pid),
+      mount_ns_instance_(new ProcMountNS(pid)), loaded_(false) {
+  std::string full_path = resolve_bin_path(bin_path);
+  if (!full_path.empty()) {
+    if (bcc_elf_foreach_usdt(full_path.c_str(), _each_probe, this) == 0) {
+      cmd_bin_path_ = ebpf::get_pid_exe(pid);
+      if (cmd_bin_path_.empty())
+        return;
+      loaded_ = true;
+    }
+  }
+  for (const auto &probe : probes_)
+    probe->finalize_locations();
+}
+
 Context::~Context() {
   if (pid_stat_ && !pid_stat_->is_stale()) {
     for (auto &p : probes_) p->disable();
@@ -341,8 +390,13 @@ Context::~Context() {
 
 extern "C" {
 
-void *bcc_usdt_new_frompid(int pid) {
-  USDT::Context *ctx = new USDT::Context(pid);
+void *bcc_usdt_new_frompid(int pid, const char *path) {
+  USDT::Context *ctx;
+
+  if (!path)
+    ctx = new USDT::Context(pid);
+  else
+    ctx = new USDT::Context(pid, path);
   if (!ctx->loaded()) {
     delete ctx;
     return nullptr;
@@ -417,23 +471,26 @@ void bcc_usdt_foreach(void *usdt, bcc_usdt_cb callback) {
   ctx->each(callback);
 }
 
-int bcc_usdt_get_location(void *usdt, const char *probe_name,
+int bcc_usdt_get_location(void *usdt, const char *provider_name,
+                          const char *probe_name,
                           int index, struct bcc_usdt_location *location) {
   USDT::Context *ctx = static_cast<USDT::Context *>(usdt);
-  USDT::Probe *probe = ctx->get(probe_name);
+  USDT::Probe *probe = ctx->get(provider_name, probe_name);
   if (!probe)
     return -1;
   if (index < 0 || (size_t)index >= probe->num_locations())
     return -1;
   location->address = probe->address(index);
+  location->bin_path = probe->location_bin_path(index);
   return 0;
 }
 
-int bcc_usdt_get_argument(void *usdt, const char *probe_name,
+int bcc_usdt_get_argument(void *usdt, const char *provider_name,
+                          const char *probe_name,
                           int location_index, int argument_index,
                           struct bcc_usdt_argument *argument) {
   USDT::Context *ctx = static_cast<USDT::Context *>(usdt);
-  USDT::Probe *probe = ctx->get(probe_name);
+  USDT::Probe *probe = ctx->get(provider_name, probe_name);
   if (!probe)
     return -1;
   if (argument_index < 0 || (size_t)argument_index >= probe->num_arguments())

src/python/bcc/libbcc.py

@@ -179,7 +179,7 @@ lib.bcc_symcache_refresh.restype = None
 lib.bcc_symcache_refresh.argtypes = [ct.c_void_p]
 
 lib.bcc_usdt_new_frompid.restype = ct.c_void_p
-lib.bcc_usdt_new_frompid.argtypes = [ct.c_int]
+lib.bcc_usdt_new_frompid.argtypes = [ct.c_int, ct.c_char_p]
 
 lib.bcc_usdt_new_frompath.restype = ct.c_void_p
 lib.bcc_usdt_new_frompath.argtypes = [ct.c_char_p]
@@ -208,7 +208,8 @@ class bcc_usdt(ct.Structure):
 
 class bcc_usdt_location(ct.Structure):
     _fields_ = [
-            ('address', ct.c_ulonglong)
+            ('address', ct.c_ulonglong),
+            ('bin_path', ct.c_char_p),
         ]
 
 class BCC_USDT_ARGUMENT_FLAGS(object):
@@ -238,11 +239,11 @@ lib.bcc_usdt_foreach.restype = None
 lib.bcc_usdt_foreach.argtypes = [ct.c_void_p, _USDT_CB]
 
 lib.bcc_usdt_get_location.restype = ct.c_int
-lib.bcc_usdt_get_location.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_int,
+lib.bcc_usdt_get_location.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_char_p, ct.c_int,
                                       ct.POINTER(bcc_usdt_location)]
 
 lib.bcc_usdt_get_argument.restype = ct.c_int
-lib.bcc_usdt_get_argument.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_int,
+lib.bcc_usdt_get_argument.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_char_p, ct.c_int,
                                       ct.c_int, ct.POINTER(bcc_usdt_argument)]
 
 _USDT_PROBE_CB = ct.CFUNCTYPE(None, ct.c_char_p, ct.c_char_p,

src/python/bcc/usdt.py

@@ -82,13 +82,15 @@ class USDTProbeLocation(object):
         self.index = index
         self.num_arguments = probe.num_arguments
         self.address = location.address
+        self.bin_path = location.bin_path
 
     def __str__(self):
-        return "0x%x" % self.address
+        return "%s 0x%x" % (self.bin_path, self.address)
 
     def get_argument(self, index):
         arg = bcc_usdt_argument()
-        res = lib.bcc_usdt_get_argument(self.probe.context, self.probe.name,
+        res = lib.bcc_usdt_get_argument(self.probe.context, self.probe.provider,
+                                        self.probe.name,
                                         self.index, index, ct.byref(arg))
         if res != 0:
             raise USDTException(
@@ -107,15 +109,15 @@ class USDTProbe(object):
         self.num_arguments = probe.num_arguments
 
     def __str__(self):
-        return "%s %s:%s [sema 0x%x]" % \
-               (self.bin_path, self.provider, self.name, self.semaphore)
+        return "%s:%s [sema 0x%x]" % \
+               (self.provider, self.name, self.semaphore)
 
     def short_name(self):
         return "%s:%s" % (self.provider, self.name)
 
     def get_location(self, index):
         loc = bcc_usdt_location()
-        res = lib.bcc_usdt_get_location(self.context, self.name,
+        res = lib.bcc_usdt_get_location(self.context, self.provider, self.name,
                                         index, ct.byref(loc))
         if res != 0:
             raise USDTException("error retrieving probe location %d" % index)
@@ -125,7 +127,10 @@ class USDT(object):
     def __init__(self, pid=None, path=None):
         if pid and pid != -1:
             self.pid = pid
-            self.context = lib.bcc_usdt_new_frompid(pid)
+            if path:
+                self.context = lib.bcc_usdt_new_frompid(pid, path.encode('ascii'))
+            else:
+                self.context = lib.bcc_usdt_new_frompid(pid, ct.c_char_p(0))
             if self.context == None:
                 raise USDTException("USDT failed to instrument PID %d" % pid)
         elif path:

tests/cc/test_usdt_probes.cc

@@ -42,7 +42,7 @@ TEST_CASE("test finding a probe in our own process", "[usdt]") {
     auto probe = ctx.get("sample_probe_1");
     REQUIRE(probe);
 
-    REQUIRE(probe->in_shared_object() == false);
+    REQUIRE(probe->in_shared_object(probe->bin_path()) == false);
     REQUIRE(probe->name() == "sample_probe_1");
     REQUIRE(probe->provider() == "libbcc_test");
     REQUIRE(probe->bin_path().find("/test_libbcc") != std::string::npos);
@@ -132,7 +132,7 @@ TEST_CASE("test listing all USDT probes in Ruby/MRI", "[usdt]") {
       auto probe = ctx.get(name);
       REQUIRE(probe);
 
-      REQUIRE(probe->in_shared_object() == true);
+      REQUIRE(probe->in_shared_object(probe->bin_path()) == true);
       REQUIRE(probe->name() == name);
       REQUIRE(probe->provider() == "ruby");
 
@@ -155,7 +155,7 @@ TEST_CASE("test listing all USDT probes in Ruby/MRI", "[usdt]") {
       auto probe = ctx.get(name);
       REQUIRE(probe);
 
-      REQUIRE(probe->in_shared_object() == true);
+      REQUIRE(probe->in_shared_object(probe->bin_path()) == true);
       REQUIRE(probe->name() == name);
       REQUIRE(probe->provider() == "ruby");
 
@@ -205,7 +205,7 @@ TEST_CASE("test listing all USDT probes in Ruby/MRI", "[usdt]") {
       auto probe = ctx.get(name);
       REQUIRE(probe);
 
-      REQUIRE(probe->in_shared_object() == true);
+      REQUIRE(probe->in_shared_object(probe->bin_path()) == true);
       REQUIRE(probe->name() == name);
       REQUIRE(probe->provider() == "ruby");
 

tests/python/CMakeLists.txt

@@ -72,3 +72,5 @@ add_test(NAME py_test_usdt WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_test_usdt sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_usdt.py)
 add_test(NAME py_test_usdt2 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_test_usdt2 sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_usdt2.py)
+add_test(NAME py_test_usdt3 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND ${TEST_WRAPPER} py_test_usdt3 sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_usdt3.py)

tests/python/test_usdt3.py

+#!/usr/bin/python
+#
+# USAGE: test_usdt3.py
+#
+# Copyright 2018 Facebook, Inc
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+from __future__ import print_function
+from bcc import BPF, USDT
+from unittest import main, TestCase
+from subprocess import Popen, PIPE
+import ctypes as ct
+import inspect, os, tempfile
+
+class TestUDST(TestCase):
+    def setUp(self):
+        common_h = b"""
+#include "folly/tracing/StaticTracepoint.h"
+
+static inline void record_val(int val)
+{
+  FOLLY_SDT(test, probe, val);
+}
+
+extern void record_a(int val);
+extern void record_b(int val);
+"""
+
+        a_c = b"""
+#include <stdio.h>
+#include "common.h"
+
+void record_a(int val)
+{
+    record_val(val);
+}
+"""
+
+        b_c = b"""
+#include <stdio.h>
+#include "common.h"
+
+void record_b(int val)
+{
+    record_val(val);
+}
+"""
+
+        m_c = b"""
+#include <stdio.h>
+#include <unistd.h>
+#include "common.h"
+
+int main() {
+   while (1) {
+     record_a(1);
+     record_b(2);
+     record_val(3);
+     sleep(1);
+   }
+   return 0;
+}
+"""
+        # BPF program
+        self.bpf_text = """
+BPF_PERF_OUTPUT(event);
+int do_trace(struct pt_regs *ctx) {
+    int result = 0;
+    bpf_usdt_readarg(1, ctx, &result);
+    event.perf_submit(ctx, &result, sizeof(result));
+    return 0;
+};
+"""
+
+        def _create_file(name, text):
+            text_file = open(name, "w")
+            text_file.write(text)
+            text_file.close()
+
+        # Create source files
+        self.tmp_dir = tempfile.mkdtemp()
+        print("temp directory: " + self.tmp_dir)
+        _create_file(self.tmp_dir + "/common.h", common_h)
+        _create_file(self.tmp_dir + "/a.c", a_c)
+        _create_file(self.tmp_dir + "/b.c", b_c)
+        _create_file(self.tmp_dir + "/m.c", m_c)
+
+        # Compilation
+        # the usdt test:probe exists in liba.so, libb.so and a.out
+        include_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + "/include"
+        a_src = self.tmp_dir + "/a.c"
+        a_obj = self.tmp_dir + "/a.o"
+        a_lib = self.tmp_dir + "/liba.so"
+        b_src = self.tmp_dir + "/b.c"
+        b_obj = self.tmp_dir + "/b.o"
+        b_lib = self.tmp_dir + "/libb.so"
+        m_src = self.tmp_dir + "/m.c"
+        m_bin = self.tmp_dir + "/a.out"
+        m_linker_opt = " -L" + self.tmp_dir + " -la -lb"
+        self.assertEqual(os.system("gcc -I" + include_path + " -fpic -c -o " + a_obj + " " + a_src), 0)
+        self.assertEqual(os.system("gcc -I" + include_path + " -fpic -c -o " + b_obj + " " + b_src), 0)
+        self.assertEqual(os.system("gcc -shared -o " + a_lib + " " + a_obj), 0)
+        self.assertEqual(os.system("gcc -shared -o " + b_lib + " " + b_obj), 0)
+        self.assertEqual(os.system("gcc -I" + include_path + " " + m_src + " -o " + m_bin + m_linker_opt), 0)
+
+        # Run the application
+        self.app = Popen([m_bin], env=dict(os.environ, LD_LIBRARY_PATH=self.tmp_dir))
+        # os.system("tplist.py -vvv -p " + str(self.app.pid))
+
+    def test_attach1(self):
+        # enable USDT probe from given PID and verifier generated BPF programs
+        u = USDT(pid=int(self.app.pid))
+        u.enable_probe(probe="probe", fn_name="do_trace")
+        b = BPF(text=self.bpf_text, usdt_contexts=[u])
+
+        # processing events
+        self.probe_value_1 = 0
+        self.probe_value_2 = 0
+        self.probe_value_3 = 0
+        self.probe_value_other = 0
+
+        def print_event(cpu, data, size):
+            result = ct.cast(data, ct.POINTER(ct.c_int)).contents
+            if result.value == 1:
+                self.probe_value_1 = 1
+            elif result.value == 2:
+                self.probe_value_2 = 1
+            elif result.value == 3:
+                self.probe_value_3 = 1
+            else:
+                self.probe_value_other = 1
+
+        b["event"].open_perf_buffer(print_event)
+        for i in range(6):
+            b.kprobe_poll()
+
+        self.assertTrue(self.probe_value_1 != 0)
+        self.assertTrue(self.probe_value_2 != 0)
+        self.assertTrue(self.probe_value_3 != 0)
+        self.assertTrue(self.probe_value_other == 0)
+
+    def tearDown(self):
+        # kill the subprocess, clean the environment
+        self.app.kill()
+        self.app.wait()
+        os.system("rm -rf " + self.tmp_dir)
+
+if __name__ == "__main__":
+    main()

tools/tplist_example.txt

@@ -20,7 +20,7 @@ $ tplist -l basic_usdt
 The loop_iter probe sounds interesting. How many arguments are available?
 
 $ tplist '*loop_iter' -l basic_usdt -v
-/home/vagrant/basic_usdt basic_usdt:loop_iter [sema 0x601036]
+basic_usdt:loop_iter [sema 0x601036]
   2 location(s)
   2 argument(s)
 
@@ -94,22 +94,22 @@ which their values are coming from. In super-verbose mode, tplist will print
 this information (note the -vv):
 
 $ tplist -vv -l c *alloc*
-/lib64/libc.so.6 libc:memory_malloc_retry [sema 0x0]
-  location #0 0x835c0
+libc:memory_malloc_retry [sema 0x0]
+  location #0 /lib64/libc.so.6 0x835c0
     argument #0 8 unsigned bytes @ bp
-  location #1 0x83778
+  location #1 /lib64/libc.so.6 0x83778
     argument #0 8 unsigned bytes @ bp
-  location #2 0x85a50
+  location #2 /lib64/libc.so.6 0x85a50
     argument #0 8 unsigned bytes @ bp
-/lib64/libc.so.6 libc:memory_realloc_retry [sema 0x0]
-  location #0 0x84b90
+libc:memory_realloc_retry [sema 0x0]
+  location #0 /lib64/libc.so.6 0x84b90
     argument #0 8 unsigned bytes @ r13
     argument #1 8 unsigned bytes @ bp
-  location #1 0x85cf0
+  location #1 /lib64/libc.so.6 0x85cf0
     argument #0 8 unsigned bytes @ r13
     argument #1 8 unsigned bytes @ bp
-/lib64/libc.so.6 libc:memory_calloc_retry [sema 0x0]
-  location #0 0x850f0
+libc:memory_calloc_retry [sema 0x0]
+  location #0 /lib64/libc.so.6 0x850f0
     argument #0 8 unsigned bytes @ bp