Commit b9f5621c authored by Like Xu's avatar Like Xu Committed by Peter Zijlstra

perf/core: Rework guest callbacks to prepare for static_call support

To prepare for using static_calls to optimize perf's guest callbacks,
replace ->is_in_guest and ->is_user_mode with a new multiplexed hook
->state, tweak ->handle_intel_pt_intr to play nice with being called when
there is no active guest, and drop "guest" from ->get_guest_ip.

Return '0' from ->state and ->handle_intel_pt_intr to indicate "not in
guest" so that DEFINE_STATIC_CALL_RET0 can be used to define the static
calls, i.e. no callback == !guest.

[sean: extracted from static_call patch, fixed get_ip() bug, wrote changelog]
Suggested-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Originally-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarLike Xu <like.xu@linux.intel.com>
Signed-off-by: default avatarZhu Lingshan <lingshan.zhu@intel.com>
Signed-off-by: default avatarSean Christopherson <seanjc@google.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarBoris Ostrovsky <boris.ostrovsky@oracle.com>
Reviewed-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
Link: https://lore.kernel.org/r/20211111020738.2512932-7-seanjc@google.com
parent 84af21d8
...@@ -104,7 +104,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, ...@@ -104,7 +104,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
{ {
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
if (guest_cbs && guest_cbs->is_in_guest()) { if (guest_cbs && guest_cbs->state()) {
/* We don't support guest os callchain now */ /* We don't support guest os callchain now */
return; return;
} }
...@@ -152,7 +152,7 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, ...@@ -152,7 +152,7 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
struct stackframe frame; struct stackframe frame;
if (guest_cbs && guest_cbs->is_in_guest()) { if (guest_cbs && guest_cbs->state()) {
/* We don't support guest os callchain now */ /* We don't support guest os callchain now */
return; return;
} }
...@@ -165,8 +165,8 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs) ...@@ -165,8 +165,8 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs)
{ {
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
if (guest_cbs && guest_cbs->is_in_guest()) if (guest_cbs && guest_cbs->state())
return guest_cbs->get_guest_ip(); return guest_cbs->get_ip();
return instruction_pointer(regs); return instruction_pointer(regs);
} }
...@@ -174,10 +174,11 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs) ...@@ -174,10 +174,11 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs)
unsigned long perf_misc_flags(struct pt_regs *regs) unsigned long perf_misc_flags(struct pt_regs *regs)
{ {
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
unsigned int guest_state = guest_cbs ? guest_cbs->state() : 0;
int misc = 0; int misc = 0;
if (guest_cbs && guest_cbs->is_in_guest()) { if (guest_state) {
if (guest_cbs->is_user_mode()) if (guest_state & PERF_GUEST_USER)
misc |= PERF_RECORD_MISC_GUEST_USER; misc |= PERF_RECORD_MISC_GUEST_USER;
else else
misc |= PERF_RECORD_MISC_GUEST_KERNEL; misc |= PERF_RECORD_MISC_GUEST_KERNEL;
......
...@@ -13,39 +13,34 @@ ...@@ -13,39 +13,34 @@
DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available); DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available);
static int kvm_is_in_guest(void) static unsigned int kvm_guest_state(void)
{ {
return kvm_get_running_vcpu() != NULL; struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
} unsigned int state;
static int kvm_is_user_mode(void)
{
struct kvm_vcpu *vcpu;
vcpu = kvm_get_running_vcpu(); if (!vcpu)
return 0;
if (vcpu) state = PERF_GUEST_ACTIVE;
return !vcpu_mode_priv(vcpu); if (!vcpu_mode_priv(vcpu))
state |= PERF_GUEST_USER;
return 0; return state;
} }
static unsigned long kvm_get_guest_ip(void) static unsigned long kvm_get_guest_ip(void)
{ {
struct kvm_vcpu *vcpu; struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
vcpu = kvm_get_running_vcpu();
if (vcpu) if (WARN_ON_ONCE(!vcpu))
return *vcpu_pc(vcpu); return 0;
return 0; return *vcpu_pc(vcpu);
} }
static struct perf_guest_info_callbacks kvm_guest_cbs = { static struct perf_guest_info_callbacks kvm_guest_cbs = {
.is_in_guest = kvm_is_in_guest, .state = kvm_guest_state,
.is_user_mode = kvm_is_user_mode, .get_ip = kvm_get_guest_ip,
.get_guest_ip = kvm_get_guest_ip,
}; };
void kvm_perf_init(void) void kvm_perf_init(void)
......
...@@ -2772,7 +2772,7 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re ...@@ -2772,7 +2772,7 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re
struct unwind_state state; struct unwind_state state;
unsigned long addr; unsigned long addr;
if (guest_cbs && guest_cbs->is_in_guest()) { if (guest_cbs && guest_cbs->state()) {
/* TODO: We don't support guest os callchain now */ /* TODO: We don't support guest os callchain now */
return; return;
} }
...@@ -2876,7 +2876,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs ...@@ -2876,7 +2876,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs
struct stack_frame frame; struct stack_frame frame;
const struct stack_frame __user *fp; const struct stack_frame __user *fp;
if (guest_cbs && guest_cbs->is_in_guest()) { if (guest_cbs && guest_cbs->state()) {
/* TODO: We don't support guest os callchain now */ /* TODO: We don't support guest os callchain now */
return; return;
} }
...@@ -2955,8 +2955,8 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs) ...@@ -2955,8 +2955,8 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs)
{ {
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
if (guest_cbs && guest_cbs->is_in_guest()) if (guest_cbs && guest_cbs->state())
return guest_cbs->get_guest_ip(); return guest_cbs->get_ip();
return regs->ip + code_segment_base(regs); return regs->ip + code_segment_base(regs);
} }
...@@ -2964,10 +2964,11 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs) ...@@ -2964,10 +2964,11 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs)
unsigned long perf_misc_flags(struct pt_regs *regs) unsigned long perf_misc_flags(struct pt_regs *regs)
{ {
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
unsigned int guest_state = guest_cbs ? guest_cbs->state() : 0;
int misc = 0; int misc = 0;
if (guest_cbs && guest_cbs->is_in_guest()) { if (guest_state) {
if (guest_cbs->is_user_mode()) if (guest_state & PERF_GUEST_USER)
misc |= PERF_RECORD_MISC_GUEST_USER; misc |= PERF_RECORD_MISC_GUEST_USER;
else else
misc |= PERF_RECORD_MISC_GUEST_KERNEL; misc |= PERF_RECORD_MISC_GUEST_KERNEL;
......
...@@ -2906,10 +2906,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) ...@@ -2906,10 +2906,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
handled++; handled++;
guest_cbs = perf_get_guest_cbs(); guest_cbs = perf_get_guest_cbs();
if (unlikely(guest_cbs && guest_cbs->is_in_guest() && if (likely(!guest_cbs || !guest_cbs->handle_intel_pt_intr()))
guest_cbs->handle_intel_pt_intr))
guest_cbs->handle_intel_pt_intr();
else
intel_pt_interrupt(); intel_pt_interrupt();
} }
......
...@@ -1895,7 +1895,7 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); ...@@ -1895,7 +1895,7 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu); void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu);
int kvm_is_in_guest(void); unsigned int kvm_guest_state(void);
void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
u32 size); u32 size);
......
...@@ -87,7 +87,7 @@ static void kvm_perf_overflow_intr(struct perf_event *perf_event, ...@@ -87,7 +87,7 @@ static void kvm_perf_overflow_intr(struct perf_event *perf_event,
* woken up. So we should wake it, but this is impossible from * woken up. So we should wake it, but this is impossible from
* NMI context. Do it from irq work instead. * NMI context. Do it from irq work instead.
*/ */
if (!kvm_is_in_guest()) if (!kvm_guest_state())
irq_work_queue(&pmc_to_pmu(pmc)->irq_work); irq_work_queue(&pmc_to_pmu(pmc)->irq_work);
else else
kvm_make_request(KVM_REQ_PMI, pmc->vcpu); kvm_make_request(KVM_REQ_PMI, pmc->vcpu);
......
...@@ -8472,44 +8472,48 @@ static void kvm_timer_init(void) ...@@ -8472,44 +8472,48 @@ static void kvm_timer_init(void)
DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu); DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu); EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu);
int kvm_is_in_guest(void) unsigned int kvm_guest_state(void)
{ {
return __this_cpu_read(current_vcpu) != NULL; struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);
} unsigned int state;
static int kvm_is_user_mode(void) if (!vcpu)
{ return 0;
int user_mode = 3;
if (__this_cpu_read(current_vcpu)) state = PERF_GUEST_ACTIVE;
user_mode = static_call(kvm_x86_get_cpl)(__this_cpu_read(current_vcpu)); if (static_call(kvm_x86_get_cpl)(vcpu))
state |= PERF_GUEST_USER;
return user_mode != 0; return state;
} }
static unsigned long kvm_get_guest_ip(void) static unsigned long kvm_guest_get_ip(void)
{ {
unsigned long ip = 0; struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);
if (__this_cpu_read(current_vcpu)) if (WARN_ON_ONCE(!vcpu))
ip = kvm_rip_read(__this_cpu_read(current_vcpu)); return 0;
return ip; return kvm_rip_read(vcpu);
} }
static void kvm_handle_intel_pt_intr(void) static unsigned int kvm_handle_intel_pt_intr(void)
{ {
struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu); struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);
/* '0' on failure so that the !PT case can use a RET0 static call. */
if (!vcpu)
return 0;
kvm_make_request(KVM_REQ_PMI, vcpu); kvm_make_request(KVM_REQ_PMI, vcpu);
__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
(unsigned long *)&vcpu->arch.pmu.global_status); (unsigned long *)&vcpu->arch.pmu.global_status);
return 1;
} }
static struct perf_guest_info_callbacks kvm_guest_cbs = { static struct perf_guest_info_callbacks kvm_guest_cbs = {
.is_in_guest = kvm_is_in_guest, .state = kvm_guest_state,
.is_user_mode = kvm_is_user_mode, .get_ip = kvm_guest_get_ip,
.get_guest_ip = kvm_get_guest_ip,
.handle_intel_pt_intr = NULL, .handle_intel_pt_intr = NULL,
}; };
......
...@@ -413,34 +413,29 @@ int pmu_apic_update(uint32_t val) ...@@ -413,34 +413,29 @@ int pmu_apic_update(uint32_t val)
} }
/* perf callbacks */ /* perf callbacks */
static int xen_is_in_guest(void) static unsigned int xen_guest_state(void)
{ {
const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
unsigned int state = 0;
if (!xenpmu_data) { if (!xenpmu_data) {
pr_warn_once("%s: pmudata not initialized\n", __func__); pr_warn_once("%s: pmudata not initialized\n", __func__);
return 0; return state;
} }
if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF)) if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF))
return 0; return state;
return 1; state |= PERF_GUEST_ACTIVE;
}
static int xen_is_user_mode(void)
{
const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
if (!xenpmu_data) { if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) {
pr_warn_once("%s: pmudata not initialized\n", __func__); if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER)
return 0; state |= PERF_GUEST_USER;
} else if (xenpmu_data->pmu.r.regs.cpl & 3) {
state |= PERF_GUEST_USER;
} }
if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) return state;
return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER);
else
return !!(xenpmu_data->pmu.r.regs.cpl & 3);
} }
static unsigned long xen_get_guest_ip(void) static unsigned long xen_get_guest_ip(void)
...@@ -456,9 +451,8 @@ static unsigned long xen_get_guest_ip(void) ...@@ -456,9 +451,8 @@ static unsigned long xen_get_guest_ip(void)
} }
static struct perf_guest_info_callbacks xen_guest_cbs = { static struct perf_guest_info_callbacks xen_guest_cbs = {
.is_in_guest = xen_is_in_guest, .state = xen_guest_state,
.is_user_mode = xen_is_user_mode, .get_ip = xen_get_guest_ip,
.get_guest_ip = xen_get_guest_ip,
}; };
/* Convert registers from Xen's format to Linux' */ /* Convert registers from Xen's format to Linux' */
......
...@@ -26,11 +26,13 @@ ...@@ -26,11 +26,13 @@
# include <asm/local64.h> # include <asm/local64.h>
#endif #endif
#define PERF_GUEST_ACTIVE 0x01
#define PERF_GUEST_USER 0x02
struct perf_guest_info_callbacks { struct perf_guest_info_callbacks {
int (*is_in_guest)(void); unsigned int (*state)(void);
int (*is_user_mode)(void); unsigned long (*get_ip)(void);
unsigned long (*get_guest_ip)(void); unsigned int (*handle_intel_pt_intr)(void);
void (*handle_intel_pt_intr)(void);
}; };
#ifdef CONFIG_HAVE_HW_BREAKPOINT #ifdef CONFIG_HAVE_HW_BREAKPOINT
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment