Skip to content

L
linux
Commit a029a4ea authored by Thomas Richter's avatar Thomas Richter Committed by Vasily Gorbik
Browse files
Options

s390/cpumf: Allow concurrent access for CPU Measurement Counter Facility 

Commit cf6acb8b ("s390/cpumf: Add support for complete counter set extraction")
allows access to the CPU Measurement Counter Facility via character
device /dev/hwctr. The access was exclusive via this device or
via perf_event_open() system call. Only one path at a time was
permitted. The CPU Measurement Counter Facility device driver blocked
access to other processes.

This patch removes this restriction and allows concurrent access to
the CPU Measurement Counter Facility from multiple processes at the same
time via perf_event_open() SVC and via /dev/hwctr device. The access
via /dev/hwctr device is still exclusive, only one process is allowed to
access this device.

This patch
- moves the /dev/hwctr device access from file perf_cpum_cf_diag.c.
  to file perf_cpum_cf.c.
- use only one trace buffer .../s390dbf/cpum_cf.
- remove cfset_csd structure and includes its members it into the
  structure cpu_cf_events. This results in one data structure and
  simplifies the access.
- rework function familiy ctr_set_enable, ctr_set_disable, ctr_set_start
  and ctr_set_stop which operate on a counter set number.
  Now they operate on a counter set bit mask.
- move CF_DIAG event functionality to file perf_cpum_cf.c. It now
  contains the complete functionality of the CPU Measurement Counter
  Facility:
  - Performance measurement support for counters using perf stat.
  - Support for complete counter set extraction with device /dev/hwctr.
  - Support for counter set extraction event CF_DIAG attached to
    samples using perf record.
- removes file perf_cpum_cf_diag.c
Signed-off-by: default avatarThomas Richter <tmricht@linux.ibm.com>
Reviewed-by: default avatarSumanth Korikkar <sumanthk@linux.ibm.com>
Signed-off-by: default avatarVasily Gorbik <gor@linux.ibm.com>
parent 5fa2ea07
Hide whitespace changes
Inline Side-by-side
Showing with 1016 additions and 1223 deletions
arch/s390/include/asm/cpu_mcf.h
View file @ a029a4ea
......@@ -32,39 +32,22 @@ static const u64 cpumf_ctr_ctl[CPUMF_CTR_SET_MAX] = {
[CPUMF_CTR_SET_MT_DIAG] = 0x20,
};
static inline void ctr_set_enable(u64 *state, int ctr_set)
{
*state |= cpumf_ctr_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT;
}
static inline void ctr_set_disable(u64 *state, int ctr_set)
{
*state &= ~(cpumf_ctr_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT);
}
static inline void ctr_set_start(u64 *state, int ctr_set)
{
*state |= cpumf_ctr_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT;
}
static inline void ctr_set_stop(u64 *state, int ctr_set)
{
*state &= ~(cpumf_ctr_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT);
}
static inline void ctr_set_multiple_enable(u64 *state, u64 ctrsets)
static inline void ctr_set_enable(u64 *state, u64 ctrsets)
{
*state |= ctrsets << CPUMF_LCCTL_ENABLE_SHIFT;
}
static inline void ctr_set_multiple_disable(u64 *state, u64 ctrsets)
static inline void ctr_set_disable(u64 *state, u64 ctrsets)
{
*state &= ~(ctrsets << CPUMF_LCCTL_ENABLE_SHIFT);
}
static inline void ctr_set_multiple_start(u64 *state, u64 ctrsets)
static inline void ctr_set_start(u64 *state, u64 ctrsets)
{
*state |= ctrsets << CPUMF_LCCTL_ACTCTL_SHIFT;
}
static inline void ctr_set_multiple_stop(u64 *state, u64 ctrsets)
static inline void ctr_set_stop(u64 *state, u64 ctrsets)
{
*state &= ~(ctrsets << CPUMF_LCCTL_ACTCTL_SHIFT);
}
......@@ -92,8 +75,15 @@ struct cpu_cf_events {
struct cpumf_ctr_info info;
atomic_t ctr_set[CPUMF_CTR_SET_MAX];
atomic64_t alert;
u64 state;
u64 state; /* For perf_event_open SVC */
u64 dev_state; /* For /dev/hwctr */
unsigned int flags;
size_t used; /* Bytes used in data */
size_t usedss; /* Bytes used in start/stop */
unsigned char start[PAGE_SIZE]; /* Counter set at event add */
unsigned char stop[PAGE_SIZE]; /* Counter set at event delete */
unsigned char data[PAGE_SIZE]; /* Counter set at /dev/hwctr */
unsigned int sets; /* # Counter set saved in memory */
};
DECLARE_PER_CPU(struct cpu_cf_events, cpu_cf_events);
......@@ -124,4 +114,6 @@ static inline int stccm_avail(void)
size_t cpum_cf_ctrset_size(enum cpumf_ctr_set ctrset,
struct cpumf_ctr_info *info);
int cfset_online_cpu(unsigned int cpu);
int cfset_offline_cpu(unsigned int cpu);
#endif /* _ASM_S390_CPU_MCF_H */
arch/s390/kernel/Makefile
View file @ a029a4ea
......@@ -71,7 +71,6 @@ obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT) += ima_arch.o
obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_cpum_cf_common.o
obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf.o perf_cpum_sf.o
obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf_events.o perf_regs.o
obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf_diag.o
obj-$(CONFIG_TRACEPOINTS) += trace.o
obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_PGSTE)) += uv.o
......
arch/s390/kernel/perf_cpum_cf.c
View file @ a029a4ea
......@@ -2,8 +2,9 @@
/*
* Performance event support for s390x - CPU-measurement Counter Facility
*
* Copyright IBM Corp. 2012, 2019
* Copyright IBM Corp. 2012, 2021
* Author(s): Hendrik Brueckner <brueckner@linux.ibm.com>
* Thomas Richter <tmricht@linux.ibm.com>
*/
#define KMSG_COMPONENT "cpum_cf"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
......@@ -14,7 +15,223 @@
#include <linux/notifier.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/miscdevice.h>
#include <asm/cpu_mcf.h>
#include <asm/hwctrset.h>
#include <asm/debug.h>
static unsigned int cfdiag_cpu_speed; /* CPU speed for CF_DIAG trailer */
static debug_info_t *cf_dbg;
#define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */
/* interval in seconds */
/* Counter sets are stored as data stream in a page sized memory buffer and
* exported to user space via raw data attached to the event sample data.
* Each counter set starts with an eight byte header consisting of:
* - a two byte eye catcher (0xfeef)
* - a one byte counter set number
* - a two byte counter set size (indicates the number of counters in this set)
* - a three byte reserved value (must be zero) to make the header the same
* size as a counter value.
* All counter values are eight byte in size.
*
* All counter sets are followed by a 64 byte trailer.
* The trailer consists of a:
* - flag field indicating valid fields when corresponding bit set
* - the counter facility first and second version number
* - the CPU speed if nonzero
* - the time stamp the counter sets have been collected
* - the time of day (TOD) base value
* - the machine type.
*
* The counter sets are saved when the process is prepared to be executed on a
* CPU and saved again when the process is going to be removed from a CPU.
* The difference of both counter sets are calculated and stored in the event
* sample data area.
*/
struct cf_ctrset_entry { /* CPU-M CF counter set entry (8 byte) */
unsigned int def:16; /* 0-15 Data Entry Format */
unsigned int set:16; /* 16-31 Counter set identifier */
unsigned int ctr:16; /* 32-47 Number of stored counters */
unsigned int res1:16; /* 48-63 Reserved */
};
struct cf_trailer_entry { /* CPU-M CF_DIAG trailer (64 byte) */
/* 0 - 7 */
union {
struct {
unsigned int clock_base:1; /* TOD clock base set */
unsigned int speed:1; /* CPU speed set */
/* Measurement alerts */
unsigned int mtda:1; /* Loss of MT ctr. data alert */
unsigned int caca:1; /* Counter auth. change alert */
unsigned int lcda:1; /* Loss of counter data alert */
};
unsigned long flags; /* 0-63 All indicators */
};
/* 8 - 15 */
unsigned int cfvn:16; /* 64-79 Ctr First Version */
unsigned int csvn:16; /* 80-95 Ctr Second Version */
unsigned int cpu_speed:32; /* 96-127 CPU speed */
/* 16 - 23 */
unsigned long timestamp; /* 128-191 Timestamp (TOD) */
/* 24 - 55 */
union {
struct {
unsigned long progusage1;
unsigned long progusage2;
unsigned long progusage3;
unsigned long tod_base;
};
unsigned long progusage[4];
};
/* 56 - 63 */
unsigned int mach_type:16; /* Machine type */
unsigned int res1:16; /* Reserved */
unsigned int res2:32; /* Reserved */
};
/* Create the trailer data at the end of a page. */
static void cfdiag_trailer(struct cf_trailer_entry *te)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cpuid cpuid;
te->cfvn = cpuhw->info.cfvn; /* Counter version numbers */
te->csvn = cpuhw->info.csvn;
get_cpu_id(&cpuid); /* Machine type */
te->mach_type = cpuid.machine;
te->cpu_speed = cfdiag_cpu_speed;
if (te->cpu_speed)
te->speed = 1;
te->clock_base = 1; /* Save clock base */
te->tod_base = tod_clock_base.tod;
te->timestamp = get_tod_clock_fast();
}
/* Read a counter set. The counter set number determines the counter set and
* the CPUM-CF first and second version number determine the number of
* available counters in each counter set.
* Each counter set starts with header containing the counter set number and
* the number of eight byte counters.
*
* The functions returns the number of bytes occupied by this counter set
* including the header.
* If there is no counter in the counter set, this counter set is useless and
* zero is returned on this case.
*
* Note that the counter sets may not be enabled or active and the stcctm
* instruction might return error 3. Depending on error_ok value this is ok,
* for example when called from cpumf_pmu_start() call back function.
*/
static size_t cfdiag_getctrset(struct cf_ctrset_entry *ctrdata, int ctrset,
size_t room, bool error_ok)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
size_t ctrset_size, need = 0;
int rc = 3; /* Assume write failure */
ctrdata->def = CF_DIAG_CTRSET_DEF;
ctrdata->set = ctrset;
ctrdata->res1 = 0;
ctrset_size = cpum_cf_ctrset_size(ctrset, &cpuhw->info);
if (ctrset_size) { /* Save data */
need = ctrset_size * sizeof(u64) + sizeof(*ctrdata);
if (need <= room) {
rc = ctr_stcctm(ctrset, ctrset_size,
(u64 *)(ctrdata + 1));
}
if (rc != 3 || error_ok)
ctrdata->ctr = ctrset_size;
else
need = 0;
}
debug_sprintf_event(cf_dbg, 3,
"%s ctrset %d ctrset_size %zu cfvn %d csvn %d"
" need %zd rc %d\n", __func__, ctrset, ctrset_size,
cpuhw->info.cfvn, cpuhw->info.csvn, need, rc);
return need;
}
/* Read out all counter sets and save them in the provided data buffer.
* The last 64 byte host an artificial trailer entry.
*/
static size_t cfdiag_getctr(void *data, size_t sz, unsigned long auth,
bool error_ok)
{
struct cf_trailer_entry *trailer;
size_t offset = 0, done;
int i;
memset(data, 0, sz);
sz -= sizeof(*trailer); /* Always room for trailer */
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
struct cf_ctrset_entry *ctrdata = data + offset;
if (!(auth & cpumf_ctr_ctl[i]))
continue; /* Counter set not authorized */
done = cfdiag_getctrset(ctrdata, i, sz - offset, error_ok);
offset += done;
}
trailer = data + offset;
cfdiag_trailer(trailer);
return offset + sizeof(*trailer);
}
/* Calculate the difference for each counter in a counter set. */
static void cfdiag_diffctrset(u64 *pstart, u64 *pstop, int counters)
{
for (; --counters >= 0; ++pstart, ++pstop)
if (*pstop >= *pstart)
*pstop -= *pstart;
else
*pstop = *pstart - *pstop + 1;
}
/* Scan the counter sets and calculate the difference of each counter
* in each set. The result is the increment of each counter during the
* period the counter set has been activated.
*
* Return true on success.
*/
static int cfdiag_diffctr(struct cpu_cf_events *cpuhw, unsigned long auth)
{
struct cf_trailer_entry *trailer_start, *trailer_stop;
struct cf_ctrset_entry *ctrstart, *ctrstop;
size_t offset = 0;
auth &= (1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1;
do {
ctrstart = (struct cf_ctrset_entry *)(cpuhw->start + offset);
ctrstop = (struct cf_ctrset_entry *)(cpuhw->stop + offset);
if (memcmp(ctrstop, ctrstart, sizeof(*ctrstop))) {
pr_err_once("cpum_cf_diag counter set compare error "
"in set %i\n", ctrstart->set);
return 0;
}
auth &= ~cpumf_ctr_ctl[ctrstart->set];
if (ctrstart->def == CF_DIAG_CTRSET_DEF) {
cfdiag_diffctrset((u64 *)(ctrstart + 1),
(u64 *)(ctrstop + 1), ctrstart->ctr);
offset += ctrstart->ctr * sizeof(u64) +
sizeof(*ctrstart);
}
} while (ctrstart->def && auth);
/* Save time_stamp from start of event in stop's trailer */
trailer_start = (struct cf_trailer_entry *)(cpuhw->start + offset);
trailer_stop = (struct cf_trailer_entry *)(cpuhw->stop + offset);
trailer_stop->progusage[0] = trailer_start->timestamp;
return 1;
}
static enum cpumf_ctr_set get_counter_set(u64 event)
{
......@@ -34,7 +251,8 @@ static enum cpumf_ctr_set get_counter_set(u64 event)
return set;
}
static int validate_ctr_version(const struct hw_perf_event *hwc)
static int validate_ctr_version(const struct hw_perf_event *hwc,
enum cpumf_ctr_set set)
{
struct cpu_cf_events *cpuhw;
int err = 0;
......@@ -43,7 +261,7 @@ static int validate_ctr_version(const struct hw_perf_event *hwc)
cpuhw = &get_cpu_var(cpu_cf_events);
/* check required version for counter sets */
switch (hwc->config_base) {
switch (set) {
case CPUMF_CTR_SET_BASIC:
case CPUMF_CTR_SET_USER:
if (cpuhw->info.cfvn < 1)
......@@ -86,6 +304,8 @@ static int validate_ctr_version(const struct hw_perf_event *hwc)
(cpuhw->info.act_ctl & mtdiag_ctl)))
err = -EOPNOTSUPP;
break;
case CPUMF_CTR_SET_MAX:
err = -EOPNOTSUPP;
}
put_cpu_var(cpu_cf_events);
......@@ -95,7 +315,6 @@ static int validate_ctr_version(const struct hw_perf_event *hwc)
static int validate_ctr_auth(const struct hw_perf_event *hwc)
{
struct cpu_cf_events *cpuhw;
u64 ctrs_state;
int err = 0;
cpuhw = &get_cpu_var(cpu_cf_events);
......@@ -105,8 +324,7 @@ static int validate_ctr_auth(const struct hw_perf_event *hwc)
* return with -ENOENT in order to fall back to other
* PMUs that might suffice the event request.
*/
ctrs_state = cpumf_ctr_ctl[hwc->config_base];
if (!(ctrs_state & cpuhw->info.auth_ctl))
if (!(hwc->config_base & cpuhw->info.auth_ctl))
err = -ENOENT;
put_cpu_var(cpu_cf_events);
......@@ -126,7 +344,7 @@ static void cpumf_pmu_enable(struct pmu *pmu)
if (cpuhw->flags & PMU_F_ENABLED)
return;
err = lcctl(cpuhw->state);
err = lcctl(cpuhw->state | cpuhw->dev_state);
if (err) {
pr_err("Enabling the performance measuring unit "
"failed with rc=%x\n", err);
......@@ -151,6 +369,7 @@ static void cpumf_pmu_disable(struct pmu *pmu)
return;
inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);
inactive |= cpuhw->dev_state;
err = lcctl(inactive);
if (err) {
pr_err("Disabling the performance measuring unit "
......@@ -199,6 +418,14 @@ static const int cpumf_generic_events_user[] = {
[PERF_COUNT_HW_BUS_CYCLES] = -1,
};
static void cpumf_hw_inuse(void)
{
mutex_lock(&pmc_reserve_mutex);
if (atomic_inc_return(&num_events) == 1)
__kernel_cpumcf_begin();
mutex_unlock(&pmc_reserve_mutex);
}
static int __hw_perf_event_init(struct perf_event *event, unsigned int type)
{
struct perf_event_attr *attr = &event->attr;
......@@ -258,11 +485,11 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type)
/*
* Use the hardware perf event structure to store the
* counter number in the 'config' member and the counter
* set number in the 'config_base'. The counter set number
* is then later used to enable/disable the counter(s).
* set number in the 'config_base' as bit mask.
* It is later used to enable/disable the counter(s).
*/
hwc->config = ev;
hwc->config_base = set;
hwc->config_base = cpumf_ctr_ctl[set];
break;
case CPUMF_CTR_SET_MAX:
/* The counter could not be associated to a counter set */
......@@ -270,22 +497,13 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type)
}
/* Initialize for using the CPU-measurement counter facility */
if (!atomic_inc_not_zero(&num_events)) {
mutex_lock(&pmc_reserve_mutex);
if (atomic_read(&num_events) == 0 && __kernel_cpumcf_begin())
err = -EBUSY;
else
atomic_inc(&num_events);
mutex_unlock(&pmc_reserve_mutex);
}
if (err)
return err;
cpumf_hw_inuse();
event->destroy = hw_perf_event_destroy;
/* Finally, validate version and authorization of the counter set */
err = validate_ctr_auth(hwc);
if (!err)
err = validate_ctr_version(hwc);
err = validate_ctr_version(hwc, set);
return err;
}
......@@ -361,6 +579,7 @@ static void cpumf_pmu_start(struct perf_event *event, int flags)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct hw_perf_event *hwc = &event->hw;
int i;
if (!(hwc->state & PERF_HES_STOPPED))
return;
......@@ -376,29 +595,92 @@ static void cpumf_pmu_start(struct perf_event *event, int flags)
* needs to be synchronized. At this point, the counter set can be in
* the inactive or disabled state.
*/
hw_perf_event_reset(event);
if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) {
cpuhw->usedss = cfdiag_getctr(cpuhw->start,
sizeof(cpuhw->start),
hwc->config_base, true);
} else {
hw_perf_event_reset(event);
}
/* increment refcount for this counter set */
atomic_inc(&cpuhw->ctr_set[hwc->config_base]);
/* Increment refcount for counter sets */
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i)
if ((hwc->config_base & cpumf_ctr_ctl[i]))
atomic_inc(&cpuhw->ctr_set[i]);
}
/* Create perf event sample with the counter sets as raw data. The sample
* is then pushed to the event subsystem and the function checks for
* possible event overflows. If an event overflow occurs, the PMU is
* stopped.
*
* Return non-zero if an event overflow occurred.
*/
static int cfdiag_push_sample(struct perf_event *event,
struct cpu_cf_events *cpuhw)
{
struct perf_sample_data data;
struct perf_raw_record raw;
struct pt_regs regs;
int overflow;
/* Setup perf sample */
perf_sample_data_init(&data, 0, event->hw.last_period);
memset(&regs, 0, sizeof(regs));
memset(&raw, 0, sizeof(raw));
if (event->attr.sample_type & PERF_SAMPLE_CPU)
data.cpu_entry.cpu = event->cpu;
if (event->attr.sample_type & PERF_SAMPLE_RAW) {
raw.frag.size = cpuhw->usedss;
raw.frag.data = cpuhw->stop;
raw.size = raw.frag.size;
data.raw = &raw;
}
overflow = perf_event_overflow(event, &data, &regs);
debug_sprintf_event(cf_dbg, 3,
"%s event %#llx sample_type %#llx raw %d ov %d\n",
__func__, event->hw.config,
event->attr.sample_type, raw.size, overflow);
if (overflow)
event->pmu->stop(event, 0);
perf_event_update_userpage(event);
return overflow;
}
static void cpumf_pmu_stop(struct perf_event *event, int flags)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct hw_perf_event *hwc = &event->hw;
int i;
if (!(hwc->state & PERF_HES_STOPPED)) {
/* Decrement reference count for this counter set and if this
* is the last used counter in the set, clear activation
* control and set the counter set state to inactive.
*/
if (!atomic_dec_return(&cpuhw->ctr_set[hwc->config_base]))
ctr_set_stop(&cpuhw->state, hwc->config_base);
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
if (!(hwc->config_base & cpumf_ctr_ctl[i]))
continue;
if (!atomic_dec_return(&cpuhw->ctr_set[i]))
ctr_set_stop(&cpuhw->state, cpumf_ctr_ctl[i]);
}
hwc->state |= PERF_HES_STOPPED;
}
if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
hw_perf_event_update(event);
if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) {
local64_inc(&event->count);
cpuhw->usedss = cfdiag_getctr(cpuhw->stop,
sizeof(cpuhw->stop),
event->hw.config_base,
false);
if (cfdiag_diffctr(cpuhw, event->hw.config_base))
cfdiag_push_sample(event, cpuhw);
} else
hw_perf_event_update(event);
hwc->state |= PERF_HES_UPTODATE;
}
}
......@@ -419,6 +701,7 @@ static int cpumf_pmu_add(struct perf_event *event, int flags)
static void cpumf_pmu_del(struct perf_event *event, int flags)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
int i;
cpumf_pmu_stop(event, PERF_EF_UPDATE);
......@@ -430,8 +713,9 @@ static void cpumf_pmu_del(struct perf_event *event, int flags)
* clear enable control and resets all counters in a set. Therefore,
* cpumf_pmu_start() always has to reenable a counter set.
*/
if (!atomic_read(&cpuhw->ctr_set[event->hw.config_base]))
ctr_set_disable(&cpuhw->state, event->hw.config_base);
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i)
if (!atomic_read(&cpuhw->ctr_set[i]))
ctr_set_disable(&cpuhw->state, cpumf_ctr_ctl[i]);
}
/* Performance monitoring unit for s390x */
......@@ -448,6 +732,7 @@ static struct pmu cpumf_pmu = {
.read = cpumf_pmu_read,
};
static int cfset_init(void);
static int __init cpumf_pmu_init(void)
{
int rc;
......@@ -455,10 +740,689 @@ static int __init cpumf_pmu_init(void)
if (!kernel_cpumcf_avail())
return -ENODEV;
/* Setup s390dbf facility */
cf_dbg = debug_register(KMSG_COMPONENT, 2, 1, 128);
if (!cf_dbg) {
pr_err("Registration of s390dbf(cpum_cf) failed\n");
return -ENOMEM;
};
debug_register_view(cf_dbg, &debug_sprintf_view);
cpumf_pmu.attr_groups = cpumf_cf_event_group();
rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", -1);
if (rc)
if (rc) {
debug_unregister_view(cf_dbg, &debug_sprintf_view);
debug_unregister(cf_dbg);
pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc);
} else if (stccm_avail()) { /* Setup counter set device */
cfset_init();
}
return rc;
}
/* Support for the CPU Measurement Facility counter set extraction using
* device /dev/hwctr. This allows user space programs to extract complete
* counter set via normal file operations.
*/
static atomic_t cfset_opencnt = ATOMIC_INIT(0); /* Excl. access */
static DEFINE_MUTEX(cfset_ctrset_mutex);/* Synchronize access to hardware */
struct cfset_call_on_cpu_parm { /* Parm struct for smp_call_on_cpu */
unsigned int sets; /* Counter set bit mask */
atomic_t cpus_ack; /* # CPUs successfully executed func */
};
static struct cfset_request { /* CPUs and counter set bit mask */
unsigned long ctrset; /* Bit mask of counter set to read */
cpumask_t mask; /* CPU mask to read from */
} cfset_request;
static void cfset_ctrset_clear(void)
{
cpumask_clear(&cfset_request.mask);
cfset_request.ctrset = 0;
}
/* The /dev/hwctr device access uses PMU_F_IN_USE to mark the device access
* path is currently used.
* The cpu_cf_events::dev_state is used to denote counter sets in use by this
* interface. It is always or'ed in. If this interface is not active, its
* value is zero and no additional counter sets will be included.
*
* The cpu_cf_events::state is used by the perf_event_open SVC and remains
* unchanged.
*
* perf_pmu_enable() and perf_pmu_enable() and its call backs
* cpumf_pmu_enable() and cpumf_pmu_disable() are called by the
* performance measurement subsystem to enable per process
* CPU Measurement counter facility.
* The XXX_enable() and XXX_disable functions are used to turn off
* x86 performance monitoring interrupt (PMI) during scheduling.
* s390 uses these calls to temporarily stop and resume the active CPU
* counters sets during scheduling.
*
* We do allow concurrent access of perf_event_open() SVC and /dev/hwctr
* device access. The perf_event_open() SVC interface makes a lot of effort
* to only run the counters while the calling process is actively scheduled
* to run.
* When /dev/hwctr interface is also used at the same time, the counter sets
* will keep running, even when the process is scheduled off a CPU.
* However this is not a problem and does not lead to wrong counter values
* for the perf_event_open() SVC. The current counter value will be recorded
* during schedule-in. At schedule-out time the current counter value is
* extracted again and the delta is calculated and added to the event.
*/
/* Stop all counter sets via ioctl interface */
static void cfset_ioctl_off(void *parm)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cfset_call_on_cpu_parm *p = parm;
int rc;
cpuhw->dev_state = 0;
for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc)
if ((p->sets & cpumf_ctr_ctl[rc]))
atomic_dec(&cpuhw->ctr_set[rc]);
rc = lcctl(cpuhw->state); /* Keep perf_event_open counter sets */
if (rc)
pr_err("Counter set stop %#llx of /dev/%s failed rc=%i\n",
cpuhw->state, S390_HWCTR_DEVICE, rc);
cpuhw->flags &= ~PMU_F_IN_USE;
debug_sprintf_event(cf_dbg, 4, "%s rc %d state %#llx dev_state %#llx\n",
__func__, rc, cpuhw->state, cpuhw->dev_state);
}
/* Start counter sets on particular CPU */
static void cfset_ioctl_on(void *parm)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cfset_call_on_cpu_parm *p = parm;
int rc;
cpuhw->flags |= PMU_F_IN_USE;
ctr_set_enable(&cpuhw->dev_state, p->sets);
ctr_set_start(&cpuhw->dev_state, p->sets);
for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc)
if ((p->sets & cpumf_ctr_ctl[rc]))
atomic_inc(&cpuhw->ctr_set[rc]);
rc = lcctl(cpuhw->dev_state | cpuhw->state); /* Start counter sets */
if (!rc)
atomic_inc(&p->cpus_ack);
else
pr_err("Counter set start %#llx of /dev/%s failed rc=%i\n",
cpuhw->dev_state | cpuhw->state, S390_HWCTR_DEVICE, rc);
debug_sprintf_event(cf_dbg, 4, "%s rc %d state %#llx dev_state %#llx\n",
__func__, rc, cpuhw->state, cpuhw->dev_state);
}
static void cfset_release_cpu(void *p)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
int rc;
debug_sprintf_event(cf_dbg, 4, "%s state %#llx dev_state %#llx\n",
__func__, cpuhw->state, cpuhw->dev_state);
rc = lcctl(cpuhw->state); /* Keep perf_event_open counter sets */
if (rc)
pr_err("Counter set release %#llx of /dev/%s failed rc=%i\n",
cpuhw->state, S390_HWCTR_DEVICE, rc);
cpuhw->dev_state = 0;
}
/* Release function is also called when application gets terminated without
* doing a proper ioctl(..., S390_HWCTR_STOP, ...) command.
*/
static int cfset_release(struct inode *inode, struct file *file)
{
on_each_cpu(cfset_release_cpu, NULL, 1);
hw_perf_event_destroy(NULL);
cfset_ctrset_clear();
atomic_set(&cfset_opencnt, 0);
return 0;
}
static int cfset_open(struct inode *inode, struct file *file)
{
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
/* Only one user space program can open /dev/hwctr */
if (atomic_xchg(&cfset_opencnt, 1))
return -EBUSY;
cpumf_hw_inuse();
file->private_data = NULL;
/* nonseekable_open() never fails */
return nonseekable_open(inode, file);
}
static int cfset_all_stop(void)
{
struct cfset_call_on_cpu_parm p = {
.sets = cfset_request.ctrset,
};
cpumask_var_t mask;
if (!alloc_cpumask_var(&mask, GFP_KERNEL))
return -ENOMEM;
cpumask_and(mask, &cfset_request.mask, cpu_online_mask);
on_each_cpu_mask(mask, cfset_ioctl_off, &p, 1);
free_cpumask_var(mask);
return 0;
}
static int cfset_all_start(void)
{
struct cfset_call_on_cpu_parm p = {
.sets = cfset_request.ctrset,
.cpus_ack = ATOMIC_INIT(0),
};
cpumask_var_t mask;
int rc = 0;
if (!alloc_cpumask_var(&mask, GFP_KERNEL))
return -ENOMEM;
cpumask_and(mask, &cfset_request.mask, cpu_online_mask);
on_each_cpu_mask(mask, cfset_ioctl_on, &p, 1);
if (atomic_read(&p.cpus_ack) != cpumask_weight(mask)) {
on_each_cpu_mask(mask, cfset_ioctl_off, &p, 1);
rc = -EIO;
debug_sprintf_event(cf_dbg, 4, "%s CPUs missing", __func__);
}
free_cpumask_var(mask);
return rc;
}
/* Return the maximum required space for all possible CPUs in case one
* CPU will be onlined during the START, READ, STOP cycles.
* To find out the size of the counter sets, any one CPU will do. They
* all have the same counter sets.
*/
static size_t cfset_needspace(unsigned int sets)
{
struct cpu_cf_events *cpuhw = get_cpu_ptr(&cpu_cf_events);
size_t bytes = 0;
int i;
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
if (!(sets & cpumf_ctr_ctl[i]))
continue;
bytes += cpum_cf_ctrset_size(i, &cpuhw->info) * sizeof(u64) +
sizeof(((struct s390_ctrset_setdata *)0)->set) +
sizeof(((struct s390_ctrset_setdata *)0)->no_cnts);
}
bytes = sizeof(((struct s390_ctrset_read *)0)->no_cpus) + nr_cpu_ids *
(bytes + sizeof(((struct s390_ctrset_cpudata *)0)->cpu_nr) +
sizeof(((struct s390_ctrset_cpudata *)0)->no_sets));
put_cpu_ptr(&cpu_cf_events);
return bytes;
}
static int cfset_all_copy(unsigned long arg, cpumask_t *mask)
{
struct s390_ctrset_read __user *ctrset_read;
unsigned int cpu, cpus, rc;
void __user *uptr;
ctrset_read = (struct s390_ctrset_read __user *)arg;
uptr = ctrset_read->data;
for_each_cpu(cpu, mask) {
struct cpu_cf_events *cpuhw = per_cpu_ptr(&cpu_cf_events, cpu);
struct s390_ctrset_cpudata __user *ctrset_cpudata;
ctrset_cpudata = uptr;
rc = put_user(cpu, &ctrset_cpudata->cpu_nr);
rc |= put_user(cpuhw->sets, &ctrset_cpudata->no_sets);
rc |= copy_to_user(ctrset_cpudata->data, cpuhw->data,
cpuhw->used);
if (rc)
return -EFAULT;
uptr += sizeof(struct s390_ctrset_cpudata) + cpuhw->used;
cond_resched();
}
cpus = cpumask_weight(mask);
if (put_user(cpus, &ctrset_read->no_cpus))
return -EFAULT;
debug_sprintf_event(cf_dbg, 4, "%s copied %ld\n", __func__,
uptr - (void __user *)ctrset_read->data);
return 0;
}
static size_t cfset_cpuset_read(struct s390_ctrset_setdata *p, int ctrset,
int ctrset_size, size_t room)
{
size_t need = 0;
int rc = -1;
need = sizeof(*p) + sizeof(u64) * ctrset_size;
if (need <= room) {
p->set = cpumf_ctr_ctl[ctrset];
p->no_cnts = ctrset_size;
rc = ctr_stcctm(ctrset, ctrset_size, (u64 *)p->cv);
if (rc == 3) /* Nothing stored */
need = 0;
}
return need;
}
/* Read all counter sets. */
static void cfset_cpu_read(void *parm)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cfset_call_on_cpu_parm *p = parm;
int set, set_size;
size_t space;
/* No data saved yet */
cpuhw->used = 0;
cpuhw->sets = 0;
memset(cpuhw->data, 0, sizeof(cpuhw->data));
/* Scan the counter sets */
for (set = CPUMF_CTR_SET_BASIC; set < CPUMF_CTR_SET_MAX; ++set) {
struct s390_ctrset_setdata *sp = (void *)cpuhw->data +
cpuhw->used;
if (!(p->sets & cpumf_ctr_ctl[set]))
continue; /* Counter set not in list */
set_size = cpum_cf_ctrset_size(set, &cpuhw->info);
space = sizeof(cpuhw->data) - cpuhw->used;
space = cfset_cpuset_read(sp, set, set_size, space);
if (space) {
cpuhw->used += space;
cpuhw->sets += 1;
}
}
debug_sprintf_event(cf_dbg, 4, "%s sets %d used %zd\n", __func__,
cpuhw->sets, cpuhw->used);
}
static int cfset_all_read(unsigned long arg)
{
struct cfset_call_on_cpu_parm p;
cpumask_var_t mask;
int rc;
if (!alloc_cpumask_var(&mask, GFP_KERNEL))
return -ENOMEM;
p.sets = cfset_request.ctrset;
cpumask_and(mask, &cfset_request.mask, cpu_online_mask);
on_each_cpu_mask(mask, cfset_cpu_read, &p, 1);
rc = cfset_all_copy(arg, mask);
free_cpumask_var(mask);
return rc;
}
subsys_initcall(cpumf_pmu_init);
static long cfset_ioctl_read(unsigned long arg)
{
struct s390_ctrset_read read;
int ret = 0;
if (copy_from_user(&read, (char __user *)arg, sizeof(read)))
return -EFAULT;
ret = cfset_all_read(arg);
return ret;
}
static long cfset_ioctl_stop(void)
{
int ret = ENXIO;
if (cfset_request.ctrset) {
ret = cfset_all_stop();
cfset_ctrset_clear();
}
return ret;
}
static long cfset_ioctl_start(unsigned long arg)
{
struct s390_ctrset_start __user *ustart;
struct s390_ctrset_start start;
void __user *umask;
unsigned int len;
int ret = 0;
size_t need;
if (cfset_request.ctrset)
return -EBUSY;
ustart = (struct s390_ctrset_start __user *)arg;
if (copy_from_user(&start, ustart, sizeof(start)))
return -EFAULT;
if (start.version != S390_HWCTR_START_VERSION)
return -EINVAL;
if (start.counter_sets & ~(cpumf_ctr_ctl[CPUMF_CTR_SET_BASIC] |
cpumf_ctr_ctl[CPUMF_CTR_SET_USER] |
cpumf_ctr_ctl[CPUMF_CTR_SET_CRYPTO] |
cpumf_ctr_ctl[CPUMF_CTR_SET_EXT] |
cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG]))
return -EINVAL; /* Invalid counter set */
if (!start.counter_sets)
return -EINVAL; /* No counter set at all? */
cpumask_clear(&cfset_request.mask);
len = min_t(u64, start.cpumask_len, cpumask_size());
umask = (void __user *)start.cpumask;
if (copy_from_user(&cfset_request.mask, umask, len))
return -EFAULT;
if (cpumask_empty(&cfset_request.mask))
return -EINVAL;
need = cfset_needspace(start.counter_sets);
if (put_user(need, &ustart->data_bytes))
ret = -EFAULT;
if (ret)
goto out;
cfset_request.ctrset = start.counter_sets;
ret = cfset_all_start();
out:
if (ret)
cfset_ctrset_clear();
debug_sprintf_event(cf_dbg, 4, "%s sets %#lx need %ld ret %d\n",
__func__, cfset_request.ctrset, need, ret);
return ret;
}
/* Entry point to the /dev/hwctr device interface.
* The ioctl system call supports three subcommands:
* S390_HWCTR_START: Start the specified counter sets on a CPU list. The
* counter set keeps running until explicitly stopped. Returns the number
* of bytes needed to store the counter values. If another S390_HWCTR_START
* ioctl subcommand is called without a previous S390_HWCTR_STOP stop
* command, -EBUSY is returned.
* S390_HWCTR_READ: Read the counter set values from specified CPU list given
* with the S390_HWCTR_START command.
* S390_HWCTR_STOP: Stops the counter sets on the CPU list given with the
* previous S390_HWCTR_START subcommand.
*/
static long cfset_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
int ret;
get_online_cpus();
mutex_lock(&cfset_ctrset_mutex);
switch (cmd) {
case S390_HWCTR_START:
ret = cfset_ioctl_start(arg);
break;
case S390_HWCTR_STOP:
ret = cfset_ioctl_stop();
break;
case S390_HWCTR_READ:
ret = cfset_ioctl_read(arg);
break;
default:
ret = -ENOTTY;
break;
}
mutex_unlock(&cfset_ctrset_mutex);
put_online_cpus();
return ret;
}
static const struct file_operations cfset_fops = {
.owner = THIS_MODULE,
.open = cfset_open,
.release = cfset_release,
.unlocked_ioctl = cfset_ioctl,
.compat_ioctl = cfset_ioctl,
.llseek = no_llseek
};
static struct miscdevice cfset_dev = {
.name = S390_HWCTR_DEVICE,
.minor = MISC_DYNAMIC_MINOR,
.fops = &cfset_fops,
};
int cfset_online_cpu(unsigned int cpu)
{
struct cfset_call_on_cpu_parm p;
mutex_lock(&cfset_ctrset_mutex);
if (cfset_request.ctrset) {
p.sets = cfset_request.ctrset;
cfset_ioctl_on(&p);
cpumask_set_cpu(cpu, &cfset_request.mask);
}
mutex_unlock(&cfset_ctrset_mutex);
return 0;
}
int cfset_offline_cpu(unsigned int cpu)
{
struct cfset_call_on_cpu_parm p;
mutex_lock(&cfset_ctrset_mutex);
if (cfset_request.ctrset) {
p.sets = cfset_request.ctrset;
cfset_ioctl_off(&p);
cpumask_clear_cpu(cpu, &cfset_request.mask);
}
mutex_unlock(&cfset_ctrset_mutex);
return 0;
}
static void cfdiag_read(struct perf_event *event)
{
debug_sprintf_event(cf_dbg, 3, "%s event %#llx count %ld\n", __func__,
event->attr.config, local64_read(&event->count));
}
static int get_authctrsets(void)
{
struct cpu_cf_events *cpuhw;
unsigned long auth = 0;
enum cpumf_ctr_set i;
cpuhw = &get_cpu_var(cpu_cf_events);
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
if (cpuhw->info.auth_ctl & cpumf_ctr_ctl[i])
auth |= cpumf_ctr_ctl[i];
}
put_cpu_var(cpu_cf_events);
return auth;
}
/* Setup the event. Test for authorized counter sets and only include counter
* sets which are authorized at the time of the setup. Including unauthorized
* counter sets result in specification exception (and panic).
*/
static int cfdiag_event_init2(struct perf_event *event)
{
struct perf_event_attr *attr = &event->attr;
int err = 0;
/* Set sample_period to indicate sampling */
event->hw.config = attr->config;
event->hw.sample_period = attr->sample_period;
local64_set(&event->hw.period_left, event->hw.sample_period);
local64_set(&event->count, 0);
event->hw.last_period = event->hw.sample_period;
/* Add all authorized counter sets to config_base. The
* the hardware init function is either called per-cpu or just once
* for all CPUS (event->cpu == -1). This depends on the whether
* counting is started for all CPUs or on a per workload base where
* the perf event moves from one CPU to another CPU.
* Checking the authorization on any CPU is fine as the hardware
* applies the same authorization settings to all CPUs.
*/
event->hw.config_base = get_authctrsets();
/* No authorized counter sets, nothing to count/sample */
if (!event->hw.config_base)
err = -EINVAL;
debug_sprintf_event(cf_dbg, 5, "%s err %d config_base %#lx\n",
__func__, err, event->hw.config_base);
return err;
}
static int cfdiag_event_init(struct perf_event *event)
{
struct perf_event_attr *attr = &event->attr;
int err = -ENOENT;
if (event->attr.config != PERF_EVENT_CPUM_CF_DIAG ||
event->attr.type != event->pmu->type)
goto out;
/* Raw events are used to access counters directly,
* hence do not permit excludes.
* This event is useless without PERF_SAMPLE_RAW to return counter set
* values as raw data.
*/
if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv ||
!(attr->sample_type & (PERF_SAMPLE_CPU | PERF_SAMPLE_RAW))) {
err = -EOPNOTSUPP;
goto out;
}
/* Initialize for using the CPU-measurement counter facility */
cpumf_hw_inuse();
event->destroy = hw_perf_event_destroy;
err = cfdiag_event_init2(event);
if (unlikely(err))
event->destroy(event);
out:
return err;
}
/* Create cf_diag/events/CF_DIAG event sysfs file. This counter is used
* to collect the complete counter sets for a scheduled process. Target
* are complete counter sets attached as raw data to the artificial event.
* This results in complete counter sets available when a process is
* scheduled. Contains the delta of every counter while the process was
* running.
*/
CPUMF_EVENT_ATTR(CF_DIAG, CF_DIAG, PERF_EVENT_CPUM_CF_DIAG);
static struct attribute *cfdiag_events_attr[] = {
CPUMF_EVENT_PTR(CF_DIAG, CF_DIAG),
NULL,
};
PMU_FORMAT_ATTR(event, "config:0-63");
static struct attribute *cfdiag_format_attr[] = {
&format_attr_event.attr,
NULL,
};
static struct attribute_group cfdiag_events_group = {
.name = "events",
.attrs = cfdiag_events_attr,
};
static struct attribute_group cfdiag_format_group = {
.name = "format",
.attrs = cfdiag_format_attr,
};
static const struct attribute_group *cfdiag_attr_groups[] = {
&cfdiag_events_group,
&cfdiag_format_group,
NULL,
};
/* Performance monitoring unit for event CF_DIAG. Since this event
* is also started and stopped via the perf_event_open() system call, use
* the same event enable/disable call back functions. They do not
* have a pointer to the perf_event strcture as first parameter.
*
* The functions XXX_add, XXX_del, XXX_start and XXX_stop are also common.
* Reuse them and distinguish the event (always first parameter) via
* 'config' member.
*/
static struct pmu cf_diag = {
.task_ctx_nr = perf_sw_context,
.event_init = cfdiag_event_init,
.pmu_enable = cpumf_pmu_enable,
.pmu_disable = cpumf_pmu_disable,
.add = cpumf_pmu_add,
.del = cpumf_pmu_del,
.start = cpumf_pmu_start,
.stop = cpumf_pmu_stop,
.read = cfdiag_read,
.attr_groups = cfdiag_attr_groups
};
/* Calculate memory needed to store all counter sets together with header and
* trailer data. This is independent of the counter set authorization which
* can vary depending on the configuration.
*/
static size_t cfdiag_maxsize(struct cpumf_ctr_info *info)
{
size_t max_size = sizeof(struct cf_trailer_entry);
enum cpumf_ctr_set i;
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
size_t size = cpum_cf_ctrset_size(i, info);
if (size)
max_size += size * sizeof(u64) +
sizeof(struct cf_ctrset_entry);
}
return max_size;
}
/* Get the CPU speed, try sampling facility first and CPU attributes second. */
static void cfdiag_get_cpu_speed(void)
{
if (cpum_sf_avail()) { /* Sampling facility first */
struct hws_qsi_info_block si;
memset(&si, 0, sizeof(si));
if (!qsi(&si)) {
cfdiag_cpu_speed = si.cpu_speed;
return;
}
}
/* Fallback: CPU speed extract static part. Used in case
* CPU Measurement Sampling Facility is turned off.
*/
if (test_facility(34)) {
unsigned long mhz = __ecag(ECAG_CPU_ATTRIBUTE, 0);
if (mhz != -1UL)
cfdiag_cpu_speed = mhz & 0xffffffff;
}
}
static int cfset_init(void)
{
struct cpumf_ctr_info info;
size_t need;
int rc;
if (qctri(&info))
return -ENODEV;
cfdiag_get_cpu_speed();
/* Make sure the counter set data fits into predefined buffer. */
need = cfdiag_maxsize(&info);
if (need > sizeof(((struct cpu_cf_events *)0)->start)) {
pr_err("Insufficient memory for PMU(cpum_cf_diag) need=%zu\n",
need);
return -ENOMEM;
}
rc = misc_register(&cfset_dev);
if (rc) {
pr_err("Registration of /dev/%s failed rc=%i\n",
cfset_dev.name, rc);
goto out;
}
rc = perf_pmu_register(&cf_diag, "cpum_cf_diag", -1);
if (rc) {
misc_deregister(&cfset_dev);
pr_err("Registration of PMU(cpum_cf_diag) failed with rc=%i\n",
rc);
}
out:
return rc;
}
device_initcall(cpumf_pmu_init);
arch/s390/kernel/perf_cpum_cf_common.c
View file @ a029a4ea
......@@ -29,7 +29,11 @@ DEFINE_PER_CPU(struct cpu_cf_events, cpu_cf_events) = {
},
.alert = ATOMIC64_INIT(0),
.state = 0,
.dev_state = 0,
.flags = 0,
.used = 0,
.usedss = 0,
.sets = 0
};
/* Indicator whether the CPU-Measurement Counter Facility Support is ready */
static bool cpum_cf_initalized;
......@@ -96,25 +100,10 @@ bool kernel_cpumcf_avail(void)
}
EXPORT_SYMBOL(kernel_cpumcf_avail);
/* Reserve/release functions for sharing perf hardware */
static DEFINE_SPINLOCK(cpumcf_owner_lock);
static void *cpumcf_owner;
/* Initialize the CPU-measurement counter facility */
int __kernel_cpumcf_begin(void)
{
int flags = PMC_INIT;
int err = 0;
spin_lock(&cpumcf_owner_lock);
if (cpumcf_owner)
err = -EBUSY;
else
cpumcf_owner = __builtin_return_address(0);
spin_unlock(&cpumcf_owner_lock);
if (err)
return err;
on_each_cpu(cpum_cf_setup_cpu, &flags, 1);
irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
......@@ -144,10 +133,6 @@ void __kernel_cpumcf_end(void)
on_each_cpu(cpum_cf_setup_cpu, &flags, 1);
irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
spin_lock(&cpumcf_owner_lock);
cpumcf_owner = NULL;
spin_unlock(&cpumcf_owner_lock);
}
EXPORT_SYMBOL(__kernel_cpumcf_end);
......@@ -161,11 +146,13 @@ static int cpum_cf_setup(unsigned int cpu, int flags)
static int cpum_cf_online_cpu(unsigned int cpu)
{
return cpum_cf_setup(cpu, PMC_INIT);
cpum_cf_setup(cpu, PMC_INIT);
return cfset_online_cpu(cpu);
}
static int cpum_cf_offline_cpu(unsigned int cpu)
{
cfset_offline_cpu(cpu);
return cpum_cf_setup(cpu, PMC_RELEASE);
}
......
arch/s390/kernel/perf_cpum_cf_diag.c deleted 100644 → 0
View file @ 5fa2ea07
// SPDX-License-Identifier: GPL-2.0
/*
* Performance event support for s390x - CPU-measurement Counter Sets
*
* Copyright IBM Corp. 2019, 2021
* Author(s): Hendrik Brueckner <brueckner@linux.ibm.com>
* Thomas Richer <tmricht@linux.ibm.com>
*/
#define KMSG_COMPONENT "cpum_cf_diag"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/percpu.h>
#include <linux/notifier.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/processor.h>
#include <linux/miscdevice.h>
#include <linux/mutex.h>
#include <asm/ctl_reg.h>
#include <asm/irq.h>
#include <asm/cpu_mcf.h>
#include <asm/timex.h>
#include <asm/debug.h>
#include <asm/hwctrset.h>
#define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */
/* interval in seconds */
static unsigned int cf_diag_cpu_speed;
static debug_info_t *cf_diag_dbg;
struct cf_diag_csd { /* Counter set data per CPU */
size_t used; /* Bytes used in data/start */
unsigned char start[PAGE_SIZE]; /* Counter set at event start */
unsigned char data[PAGE_SIZE]; /* Counter set at event delete */
unsigned int sets; /* # Counter set saved in data */
};
static DEFINE_PER_CPU(struct cf_diag_csd, cf_diag_csd);
/* Counter sets are stored as data stream in a page sized memory buffer and
* exported to user space via raw data attached to the event sample data.
* Each counter set starts with an eight byte header consisting of:
* - a two byte eye catcher (0xfeef)
* - a one byte counter set number
* - a two byte counter set size (indicates the number of counters in this set)
* - a three byte reserved value (must be zero) to make the header the same
* size as a counter value.
* All counter values are eight byte in size.
*
* All counter sets are followed by a 64 byte trailer.
* The trailer consists of a:
* - flag field indicating valid fields when corresponding bit set
* - the counter facility first and second version number
* - the CPU speed if nonzero
* - the time stamp the counter sets have been collected
* - the time of day (TOD) base value
* - the machine type.
*
* The counter sets are saved when the process is prepared to be executed on a
* CPU and saved again when the process is going to be removed from a CPU.
* The difference of both counter sets are calculated and stored in the event
* sample data area.
*/
struct cf_ctrset_entry { /* CPU-M CF counter set entry (8 byte) */
unsigned int def:16; /* 0-15 Data Entry Format */
unsigned int set:16; /* 16-31 Counter set identifier */
unsigned int ctr:16; /* 32-47 Number of stored counters */
unsigned int res1:16; /* 48-63 Reserved */
};
struct cf_trailer_entry { /* CPU-M CF_DIAG trailer (64 byte) */
/* 0 - 7 */
union {
struct {
unsigned int clock_base:1; /* TOD clock base set */
unsigned int speed:1; /* CPU speed set */
/* Measurement alerts */
unsigned int mtda:1; /* Loss of MT ctr. data alert */
unsigned int caca:1; /* Counter auth. change alert */
unsigned int lcda:1; /* Loss of counter data alert */
};
unsigned long flags; /* 0-63 All indicators */
};
/* 8 - 15 */
unsigned int cfvn:16; /* 64-79 Ctr First Version */
unsigned int csvn:16; /* 80-95 Ctr Second Version */
unsigned int cpu_speed:32; /* 96-127 CPU speed */
/* 16 - 23 */
unsigned long timestamp; /* 128-191 Timestamp (TOD) */
/* 24 - 55 */
union {
struct {
unsigned long progusage1;
unsigned long progusage2;
unsigned long progusage3;
unsigned long tod_base;
};
unsigned long progusage[4];
};
/* 56 - 63 */
unsigned int mach_type:16; /* Machine type */
unsigned int res1:16; /* Reserved */
unsigned int res2:32; /* Reserved */
};
/* Create the trailer data at the end of a page. */
static void cf_diag_trailer(struct cf_trailer_entry *te)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cpuid cpuid;
te->cfvn = cpuhw->info.cfvn; /* Counter version numbers */
te->csvn = cpuhw->info.csvn;
get_cpu_id(&cpuid); /* Machine type */
te->mach_type = cpuid.machine;
te->cpu_speed = cf_diag_cpu_speed;
if (te->cpu_speed)
te->speed = 1;
te->clock_base = 1; /* Save clock base */
te->tod_base = tod_clock_base.tod;
te->timestamp = get_tod_clock_fast();
}
/*
* Change the CPUMF state to active.
* Enable and activate the CPU-counter sets according
* to the per-cpu control state.
*/
static void cf_diag_enable(struct pmu *pmu)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
int err;
debug_sprintf_event(cf_diag_dbg, 5,
"%s pmu %p cpu %d flags %#x state %#llx\n",
__func__, pmu, smp_processor_id(), cpuhw->flags,
cpuhw->state);
if (cpuhw->flags & PMU_F_ENABLED)
return;
err = lcctl(cpuhw->state);
if (err) {
pr_err("Enabling the performance measuring unit "
"failed with rc=%x\n", err);
return;
}
cpuhw->flags |= PMU_F_ENABLED;
}
/*
* Change the CPUMF state to inactive.
* Disable and enable (inactive) the CPU-counter sets according
* to the per-cpu control state.
*/
static void cf_diag_disable(struct pmu *pmu)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
u64 inactive;
int err;
debug_sprintf_event(cf_diag_dbg, 5,
"%s pmu %p cpu %d flags %#x state %#llx\n",
__func__, pmu, smp_processor_id(), cpuhw->flags,
cpuhw->state);
if (!(cpuhw->flags & PMU_F_ENABLED))
return;
inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);
err = lcctl(inactive);
if (err) {
pr_err("Disabling the performance measuring unit "
"failed with rc=%x\n", err);
return;
}
cpuhw->flags &= ~PMU_F_ENABLED;
}
/* Number of perf events counting hardware events */
static atomic_t cf_diag_events = ATOMIC_INIT(0);
/* Used to avoid races in calling reserve/release_cpumf_hardware */
static DEFINE_MUTEX(cf_diag_reserve_mutex);
/* Release the PMU if event is the last perf event */
static void cf_diag_perf_event_destroy(struct perf_event *event)
{
debug_sprintf_event(cf_diag_dbg, 5,
"%s event %p cpu %d cf_diag_events %d\n",
__func__, event, smp_processor_id(),
atomic_read(&cf_diag_events));
if (atomic_dec_return(&cf_diag_events) == 0)
__kernel_cpumcf_end();
}
static int get_authctrsets(void)
{
struct cpu_cf_events *cpuhw;
unsigned long auth = 0;
enum cpumf_ctr_set i;
cpuhw = &get_cpu_var(cpu_cf_events);
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
if (cpuhw->info.auth_ctl & cpumf_ctr_ctl[i])
auth |= cpumf_ctr_ctl[i];
}
put_cpu_var(cpu_cf_events);
return auth;
}
/* Setup the event. Test for authorized counter sets and only include counter
* sets which are authorized at the time of the setup. Including unauthorized
* counter sets result in specification exception (and panic).
*/
static int __hw_perf_event_init(struct perf_event *event)
{
struct perf_event_attr *attr = &event->attr;
int err = 0;
debug_sprintf_event(cf_diag_dbg, 5, "%s event %p cpu %d\n", __func__,
event, event->cpu);
event->hw.config = attr->config;
/* Add all authorized counter sets to config_base. The
* the hardware init function is either called per-cpu or just once
* for all CPUS (event->cpu == -1). This depends on the whether
* counting is started for all CPUs or on a per workload base where
* the perf event moves from one CPU to another CPU.
* Checking the authorization on any CPU is fine as the hardware
* applies the same authorization settings to all CPUs.
*/
event->hw.config_base = get_authctrsets();
/* No authorized counter sets, nothing to count/sample */
if (!event->hw.config_base) {
err = -EINVAL;
goto out;
}
/* Set sample_period to indicate sampling */
event->hw.sample_period = attr->sample_period;
local64_set(&event->hw.period_left, event->hw.sample_period);
event->hw.last_period = event->hw.sample_period;
out:
debug_sprintf_event(cf_diag_dbg, 5, "%s err %d config_base %#lx\n",
__func__, err, event->hw.config_base);
return err;
}
/* Return 0 if the CPU-measurement counter facility is currently free
* and an error otherwise.
*/
static int cf_diag_perf_event_inuse(void)
{
int err = 0;
if (!atomic_inc_not_zero(&cf_diag_events)) {
mutex_lock(&cf_diag_reserve_mutex);
if (atomic_read(&cf_diag_events) == 0 &&
__kernel_cpumcf_begin())
err = -EBUSY;
else
err = atomic_inc_return(&cf_diag_events);
mutex_unlock(&cf_diag_reserve_mutex);
}
return err;
}
static int cf_diag_event_init(struct perf_event *event)
{
struct perf_event_attr *attr = &event->attr;
int err = -ENOENT;
debug_sprintf_event(cf_diag_dbg, 5,
"%s event %p cpu %d config %#llx type:%u "
"sample_type %#llx cf_diag_events %d\n", __func__,
event, event->cpu, attr->config, event->pmu->type,
attr->sample_type, atomic_read(&cf_diag_events));
if (event->attr.config != PERF_EVENT_CPUM_CF_DIAG ||
event->attr.type != event->pmu->type)
goto out;
/* Raw events are used to access counters directly,
* hence do not permit excludes.
* This event is usesless without PERF_SAMPLE_RAW to return counter set
* values as raw data.
*/
if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv ||
!(attr->sample_type & (PERF_SAMPLE_CPU | PERF_SAMPLE_RAW))) {
err = -EOPNOTSUPP;
goto out;
}
/* Initialize for using the CPU-measurement counter facility */
err = cf_diag_perf_event_inuse();
if (err < 0)
goto out;
event->destroy = cf_diag_perf_event_destroy;
err = __hw_perf_event_init(event);
if (unlikely(err))
event->destroy(event);
out:
debug_sprintf_event(cf_diag_dbg, 5, "%s err %d\n", __func__, err);
return err;
}
static void cf_diag_read(struct perf_event *event)
{
debug_sprintf_event(cf_diag_dbg, 5, "%s event %p\n", __func__, event);
}
/* Calculate memory needed to store all counter sets together with header and
* trailer data. This is independend of the counter set authorization which
* can vary depending on the configuration.
*/
static size_t cf_diag_ctrset_maxsize(struct cpumf_ctr_info *info)
{
size_t max_size = sizeof(struct cf_trailer_entry);
enum cpumf_ctr_set i;
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
size_t size = cpum_cf_ctrset_size(i, info);
if (size)
max_size += size * sizeof(u64) +
sizeof(struct cf_ctrset_entry);
}
debug_sprintf_event(cf_diag_dbg, 5, "%s max_size %zu\n", __func__,
max_size);
return max_size;
}
/* Read a counter set. The counter set number determines which counter set and
* the CPUM-CF first and second version number determine the number of
* available counters in this counter set.
* Each counter set starts with header containing the counter set number and
* the number of 8 byte counters.
*
* The functions returns the number of bytes occupied by this counter set
* including the header.
* If there is no counter in the counter set, this counter set is useless and
* zero is returned on this case.
*/
static size_t cf_diag_getctrset(struct cf_ctrset_entry *ctrdata, int ctrset,
size_t room)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
size_t ctrset_size, need = 0;
int rc = 3; /* Assume write failure */
ctrdata->def = CF_DIAG_CTRSET_DEF;
ctrdata->set = ctrset;
ctrdata->res1 = 0;
ctrset_size = cpum_cf_ctrset_size(ctrset, &cpuhw->info);
if (ctrset_size) { /* Save data */
need = ctrset_size * sizeof(u64) + sizeof(*ctrdata);
if (need <= room)
rc = ctr_stcctm(ctrset, ctrset_size,
(u64 *)(ctrdata + 1));
if (rc != 3)
ctrdata->ctr = ctrset_size;
else
need = 0;
}
debug_sprintf_event(cf_diag_dbg, 6,
"%s ctrset %d ctrset_size %zu cfvn %d csvn %d"
" need %zd rc %d\n",
__func__, ctrset, ctrset_size, cpuhw->info.cfvn,
cpuhw->info.csvn, need, rc);
return need;
}
/* Read out all counter sets and save them in the provided data buffer.
* The last 64 byte host an artificial trailer entry.
*/
static size_t cf_diag_getctr(void *data, size_t sz, unsigned long auth)
{
struct cf_trailer_entry *trailer;
size_t offset = 0, done;
int i;
memset(data, 0, sz);
sz -= sizeof(*trailer); /* Always room for trailer */
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
struct cf_ctrset_entry *ctrdata = data + offset;
if (!(auth & cpumf_ctr_ctl[i]))
continue; /* Counter set not authorized */
done = cf_diag_getctrset(ctrdata, i, sz - offset);
offset += done;
debug_sprintf_event(cf_diag_dbg, 6,
"%s ctrset %d offset %zu done %zu\n",
__func__, i, offset, done);
}
trailer = data + offset;
cf_diag_trailer(trailer);
return offset + sizeof(*trailer);
}
/* Calculate the difference for each counter in a counter set. */
static void cf_diag_diffctrset(u64 *pstart, u64 *pstop, int counters)
{
for (; --counters >= 0; ++pstart, ++pstop)
if (*pstop >= *pstart)
*pstop -= *pstart;
else
*pstop = *pstart - *pstop;
}
/* Scan the counter sets and calculate the difference of each counter
* in each set. The result is the increment of each counter during the
* period the counter set has been activated.
*
* Return true on success.
*/
static int cf_diag_diffctr(struct cf_diag_csd *csd, unsigned long auth)
{
struct cf_trailer_entry *trailer_start, *trailer_stop;
struct cf_ctrset_entry *ctrstart, *ctrstop;
size_t offset = 0;
auth &= (1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1;
do {
ctrstart = (struct cf_ctrset_entry *)(csd->start + offset);
ctrstop = (struct cf_ctrset_entry *)(csd->data + offset);
if (memcmp(ctrstop, ctrstart, sizeof(*ctrstop))) {
pr_err("cpum_cf_diag counter set compare error "
"in set %i\n", ctrstart->set);
return 0;
}
auth &= ~cpumf_ctr_ctl[ctrstart->set];
if (ctrstart->def == CF_DIAG_CTRSET_DEF) {
cf_diag_diffctrset((u64 *)(ctrstart + 1),
(u64 *)(ctrstop + 1), ctrstart->ctr);
offset += ctrstart->ctr * sizeof(u64) +
sizeof(*ctrstart);
}
debug_sprintf_event(cf_diag_dbg, 6,
"%s set %d ctr %d offset %zu auth %lx\n",
__func__, ctrstart->set, ctrstart->ctr,
offset, auth);
} while (ctrstart->def && auth);
/* Save time_stamp from start of event in stop's trailer */
trailer_start = (struct cf_trailer_entry *)(csd->start + offset);
trailer_stop = (struct cf_trailer_entry *)(csd->data + offset);
trailer_stop->progusage[0] = trailer_start->timestamp;
return 1;
}
/* Create perf event sample with the counter sets as raw data. The sample
* is then pushed to the event subsystem and the function checks for
* possible event overflows. If an event overflow occurs, the PMU is
* stopped.
*
* Return non-zero if an event overflow occurred.
*/
static int cf_diag_push_sample(struct perf_event *event,
struct cf_diag_csd *csd)
{
struct perf_sample_data data;
struct perf_raw_record raw;
struct pt_regs regs;
int overflow;
/* Setup perf sample */
perf_sample_data_init(&data, 0, event->hw.last_period);
memset(&regs, 0, sizeof(regs));
memset(&raw, 0, sizeof(raw));
if (event->attr.sample_type & PERF_SAMPLE_CPU)
data.cpu_entry.cpu = event->cpu;
if (event->attr.sample_type & PERF_SAMPLE_RAW) {
raw.frag.size = csd->used;
raw.frag.data = csd->data;
raw.size = csd->used;
data.raw = &raw;
}
overflow = perf_event_overflow(event, &data, &regs);
debug_sprintf_event(cf_diag_dbg, 6,
"%s event %p cpu %d sample_type %#llx raw %d "
"ov %d\n", __func__, event, event->cpu,
event->attr.sample_type, raw.size, overflow);
if (overflow)
event->pmu->stop(event, 0);
perf_event_update_userpage(event);
return overflow;
}
static void cf_diag_start(struct perf_event *event, int flags)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cf_diag_csd *csd = this_cpu_ptr(&cf_diag_csd);
struct hw_perf_event *hwc = &event->hw;
debug_sprintf_event(cf_diag_dbg, 5,
"%s event %p cpu %d flags %#x hwc-state %#x\n",
__func__, event, event->cpu, flags, hwc->state);
if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
return;
/* (Re-)enable and activate all counter sets */
lcctl(0); /* Reset counter sets */
hwc->state = 0;
ctr_set_multiple_enable(&cpuhw->state, hwc->config_base);
lcctl(cpuhw->state); /* Enable counter sets */
csd->used = cf_diag_getctr(csd->start, sizeof(csd->start),
event->hw.config_base);
ctr_set_multiple_start(&cpuhw->state, hwc->config_base);
/* Function cf_diag_enable() starts the counter sets. */
}
static void cf_diag_stop(struct perf_event *event, int flags)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cf_diag_csd *csd = this_cpu_ptr(&cf_diag_csd);
struct hw_perf_event *hwc = &event->hw;
debug_sprintf_event(cf_diag_dbg, 5,
"%s event %p cpu %d flags %#x hwc-state %#x\n",
__func__, event, event->cpu, flags, hwc->state);
/* Deactivate all counter sets */
ctr_set_multiple_stop(&cpuhw->state, hwc->config_base);
local64_inc(&event->count);
csd->used = cf_diag_getctr(csd->data, sizeof(csd->data),
event->hw.config_base);
if (cf_diag_diffctr(csd, event->hw.config_base))
cf_diag_push_sample(event, csd);
hwc->state |= PERF_HES_STOPPED;
}
static int cf_diag_add(struct perf_event *event, int flags)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
int err = 0;
debug_sprintf_event(cf_diag_dbg, 5,
"%s event %p cpu %d flags %#x cpuhw %p\n",
__func__, event, event->cpu, flags, cpuhw);
if (cpuhw->flags & PMU_F_IN_USE) {
err = -EAGAIN;
goto out;
}
event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
cpuhw->flags |= PMU_F_IN_USE;
if (flags & PERF_EF_START)
cf_diag_start(event, PERF_EF_RELOAD);
out:
debug_sprintf_event(cf_diag_dbg, 5, "%s err %d\n", __func__, err);
return err;
}
static void cf_diag_del(struct perf_event *event, int flags)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
debug_sprintf_event(cf_diag_dbg, 5,
"%s event %p cpu %d flags %#x\n",
__func__, event, event->cpu, flags);
cf_diag_stop(event, PERF_EF_UPDATE);
ctr_set_multiple_stop(&cpuhw->state, event->hw.config_base);
ctr_set_multiple_disable(&cpuhw->state, event->hw.config_base);
cpuhw->flags &= ~PMU_F_IN_USE;
}
/* Default counter set events and format attribute groups */
CPUMF_EVENT_ATTR(CF_DIAG, CF_DIAG, PERF_EVENT_CPUM_CF_DIAG);
static struct attribute *cf_diag_events_attr[] = {
CPUMF_EVENT_PTR(CF_DIAG, CF_DIAG),
NULL,
};
PMU_FORMAT_ATTR(event, "config:0-63");
static struct attribute *cf_diag_format_attr[] = {
&format_attr_event.attr,
NULL,
};
static struct attribute_group cf_diag_events_group = {
.name = "events",
.attrs = cf_diag_events_attr,
};
static struct attribute_group cf_diag_format_group = {
.name = "format",
.attrs = cf_diag_format_attr,
};
static const struct attribute_group *cf_diag_attr_groups[] = {
&cf_diag_events_group,
&cf_diag_format_group,
NULL,
};
/* Performance monitoring unit for s390x */
static struct pmu cf_diag = {
.task_ctx_nr = perf_sw_context,
.pmu_enable = cf_diag_enable,
.pmu_disable = cf_diag_disable,
.event_init = cf_diag_event_init,
.add = cf_diag_add,
.del = cf_diag_del,
.start = cf_diag_start,
.stop = cf_diag_stop,
.read = cf_diag_read,
.attr_groups = cf_diag_attr_groups
};
/* Get the CPU speed, try sampling facility first and CPU attributes second. */
static void cf_diag_get_cpu_speed(void)
{
if (cpum_sf_avail()) { /* Sampling facility first */
struct hws_qsi_info_block si;
memset(&si, 0, sizeof(si));
if (!qsi(&si)) {
cf_diag_cpu_speed = si.cpu_speed;
return;
}
}
if (test_facility(34)) { /* CPU speed extract static part */
unsigned long mhz = __ecag(ECAG_CPU_ATTRIBUTE, 0);
if (mhz != -1UL)
cf_diag_cpu_speed = mhz & 0xffffffff;
}
}
/* Code to create device and file I/O operations */
static atomic_t ctrset_opencnt = ATOMIC_INIT(0); /* Excl. access */
static int cf_diag_open(struct inode *inode, struct file *file)
{
int err = 0;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (atomic_xchg(&ctrset_opencnt, 1))
return -EBUSY;
/* Avoid concurrent access with perf_event_open() system call */
mutex_lock(&cf_diag_reserve_mutex);
if (atomic_read(&cf_diag_events) || __kernel_cpumcf_begin())
err = -EBUSY;
mutex_unlock(&cf_diag_reserve_mutex);
if (err) {
atomic_set(&ctrset_opencnt, 0);
return err;
}
file->private_data = NULL;
debug_sprintf_event(cf_diag_dbg, 2, "%s\n", __func__);
/* nonseekable_open() never fails */
return nonseekable_open(inode, file);
}
/* Variables for ioctl() interface support */
static DEFINE_MUTEX(cf_diag_ctrset_mutex);
static struct cf_diag_ctrset {
unsigned long ctrset; /* Bit mask of counter set to read */
cpumask_t mask; /* CPU mask to read from */
} cf_diag_ctrset;
static void cf_diag_ctrset_clear(void)
{
cpumask_clear(&cf_diag_ctrset.mask);
cf_diag_ctrset.ctrset = 0;
}
static void cf_diag_release_cpu(void *p)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
debug_sprintf_event(cf_diag_dbg, 3, "%s cpu %d\n", __func__,
smp_processor_id());
lcctl(0); /* Reset counter sets */
cpuhw->state = 0; /* Save state in CPU hardware state */
}
/* Release function is also called when application gets terminated without
* doing a proper ioctl(..., S390_HWCTR_STOP, ...) command.
* Since only one application is allowed to open the device, simple stop all
* CPU counter sets.
*/
static int cf_diag_release(struct inode *inode, struct file *file)
{
on_each_cpu(cf_diag_release_cpu, NULL, 1);
cf_diag_ctrset_clear();
atomic_set(&ctrset_opencnt, 0);
__kernel_cpumcf_end();
debug_sprintf_event(cf_diag_dbg, 2, "%s\n", __func__);
return 0;
}
struct cf_diag_call_on_cpu_parm { /* Parm struct for smp_call_on_cpu */
unsigned int sets; /* Counter set bit mask */
atomic_t cpus_ack; /* # CPUs successfully executed func */
};
static int cf_diag_all_copy(unsigned long arg, cpumask_t *mask)
{
struct s390_ctrset_read __user *ctrset_read;
unsigned int cpu, cpus, rc;
void __user *uptr;
ctrset_read = (struct s390_ctrset_read __user *)arg;
uptr = ctrset_read->data;
for_each_cpu(cpu, mask) {
struct cf_diag_csd *csd = per_cpu_ptr(&cf_diag_csd, cpu);
struct s390_ctrset_cpudata __user *ctrset_cpudata;
ctrset_cpudata = uptr;
debug_sprintf_event(cf_diag_dbg, 5, "%s cpu %d used %zd\n",
__func__, cpu, csd->used);
rc = put_user(cpu, &ctrset_cpudata->cpu_nr);
rc |= put_user(csd->sets, &ctrset_cpudata->no_sets);
rc |= copy_to_user(ctrset_cpudata->data, csd->data, csd->used);
if (rc)
return -EFAULT;
uptr += sizeof(struct s390_ctrset_cpudata) + csd->used;
cond_resched();
}
cpus = cpumask_weight(mask);
if (put_user(cpus, &ctrset_read->no_cpus))
return -EFAULT;
debug_sprintf_event(cf_diag_dbg, 5, "%s copied %ld\n",
__func__, uptr - (void __user *)ctrset_read->data);
return 0;
}
static size_t cf_diag_cpuset_read(struct s390_ctrset_setdata *p, int ctrset,
int ctrset_size, size_t room)
{
size_t need = 0;
int rc = -1;
need = sizeof(*p) + sizeof(u64) * ctrset_size;
debug_sprintf_event(cf_diag_dbg, 5,
"%s room %zd need %zd set %#x set_size %d\n",
__func__, room, need, ctrset, ctrset_size);
if (need <= room) {
p->set = cpumf_ctr_ctl[ctrset];
p->no_cnts = ctrset_size;
rc = ctr_stcctm(ctrset, ctrset_size, (u64 *)p->cv);
if (rc == 3) /* Nothing stored */
need = 0;
}
debug_sprintf_event(cf_diag_dbg, 5, "%s need %zd rc %d\n", __func__,
need, rc);
return need;
}
/* Read all counter sets. Since the perf_event_open() system call with
* event cpum_cf_diag/.../ is blocked when this interface is active, reuse
* the perf_event_open() data buffer to store the counter sets.
*/
static void cf_diag_cpu_read(void *parm)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cf_diag_csd *csd = this_cpu_ptr(&cf_diag_csd);
struct cf_diag_call_on_cpu_parm *p = parm;
int set, set_size;
size_t space;
debug_sprintf_event(cf_diag_dbg, 5,
"%s new %#x flags %#x state %#llx\n",
__func__, p->sets, cpuhw->flags,
cpuhw->state);
/* No data saved yet */
csd->used = 0;
csd->sets = 0;
memset(csd->data, 0, sizeof(csd->data));
/* Scan the counter sets */
for (set = CPUMF_CTR_SET_BASIC; set < CPUMF_CTR_SET_MAX; ++set) {
struct s390_ctrset_setdata *sp = (void *)csd->data + csd->used;
if (!(p->sets & cpumf_ctr_ctl[set]))
continue; /* Counter set not in list */
set_size = cpum_cf_ctrset_size(set, &cpuhw->info);
space = sizeof(csd->data) - csd->used;
space = cf_diag_cpuset_read(sp, set, set_size, space);
if (space) {
csd->used += space;
csd->sets += 1;
}
debug_sprintf_event(cf_diag_dbg, 5, "%s sp %px space %zd\n",
__func__, sp, space);
}
debug_sprintf_event(cf_diag_dbg, 5, "%s sets %d used %zd\n", __func__,
csd->sets, csd->used);
}
static int cf_diag_all_read(unsigned long arg)
{
struct cf_diag_call_on_cpu_parm p;
cpumask_var_t mask;
int rc;
debug_sprintf_event(cf_diag_dbg, 5, "%s\n", __func__);
if (!alloc_cpumask_var(&mask, GFP_KERNEL))
return -ENOMEM;
p.sets = cf_diag_ctrset.ctrset;
cpumask_and(mask, &cf_diag_ctrset.mask, cpu_online_mask);
on_each_cpu_mask(mask, cf_diag_cpu_read, &p, 1);
rc = cf_diag_all_copy(arg, mask);
free_cpumask_var(mask);
debug_sprintf_event(cf_diag_dbg, 5, "%s rc %d\n", __func__, rc);
return rc;
}
/* Stop all counter sets via ioctl interface */
static void cf_diag_ioctl_off(void *parm)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cf_diag_call_on_cpu_parm *p = parm;
int rc;
debug_sprintf_event(cf_diag_dbg, 5,
"%s new %#x flags %#x state %#llx\n",
__func__, p->sets, cpuhw->flags,
cpuhw->state);
ctr_set_multiple_disable(&cpuhw->state, p->sets);
ctr_set_multiple_stop(&cpuhw->state, p->sets);
rc = lcctl(cpuhw->state); /* Stop counter sets */
if (!cpuhw->state)
cpuhw->flags &= ~PMU_F_IN_USE;
debug_sprintf_event(cf_diag_dbg, 5,
"%s rc %d flags %#x state %#llx\n", __func__,
rc, cpuhw->flags, cpuhw->state);
}
/* Start counter sets on particular CPU */
static void cf_diag_ioctl_on(void *parm)
{
struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
struct cf_diag_call_on_cpu_parm *p = parm;
int rc;
debug_sprintf_event(cf_diag_dbg, 5,
"%s new %#x flags %#x state %#llx\n",
__func__, p->sets, cpuhw->flags,
cpuhw->state);
if (!(cpuhw->flags & PMU_F_IN_USE))
cpuhw->state = 0;
cpuhw->flags |= PMU_F_IN_USE;
rc = lcctl(cpuhw->state); /* Reset unused counter sets */
ctr_set_multiple_enable(&cpuhw->state, p->sets);
ctr_set_multiple_start(&cpuhw->state, p->sets);
rc |= lcctl(cpuhw->state); /* Start counter sets */
if (!rc)
atomic_inc(&p->cpus_ack);
debug_sprintf_event(cf_diag_dbg, 5, "%s rc %d state %#llx\n",
__func__, rc, cpuhw->state);
}
static int cf_diag_all_stop(void)
{
struct cf_diag_call_on_cpu_parm p = {
.sets = cf_diag_ctrset.ctrset,
};
cpumask_var_t mask;
if (!alloc_cpumask_var(&mask, GFP_KERNEL))
return -ENOMEM;
cpumask_and(mask, &cf_diag_ctrset.mask, cpu_online_mask);
on_each_cpu_mask(mask, cf_diag_ioctl_off, &p, 1);
free_cpumask_var(mask);
return 0;
}
static int cf_diag_all_start(void)
{
struct cf_diag_call_on_cpu_parm p = {
.sets = cf_diag_ctrset.ctrset,
.cpus_ack = ATOMIC_INIT(0),
};
cpumask_var_t mask;
int rc = 0;
if (!alloc_cpumask_var(&mask, GFP_KERNEL))
return -ENOMEM;
cpumask_and(mask, &cf_diag_ctrset.mask, cpu_online_mask);
on_each_cpu_mask(mask, cf_diag_ioctl_on, &p, 1);
if (atomic_read(&p.cpus_ack) != cpumask_weight(mask)) {
on_each_cpu_mask(mask, cf_diag_ioctl_off, &p, 1);
rc = -EIO;
}
free_cpumask_var(mask);
return rc;
}
/* Return the maximum required space for all possible CPUs in case one
* CPU will be onlined during the START, READ, STOP cycles.
* To find out the size of the counter sets, any one CPU will do. They
* all have the same counter sets.
*/
static size_t cf_diag_needspace(unsigned int sets)
{
struct cpu_cf_events *cpuhw = get_cpu_ptr(&cpu_cf_events);
size_t bytes = 0;
int i;
for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
if (!(sets & cpumf_ctr_ctl[i]))
continue;
bytes += cpum_cf_ctrset_size(i, &cpuhw->info) * sizeof(u64) +
sizeof(((struct s390_ctrset_setdata *)0)->set) +
sizeof(((struct s390_ctrset_setdata *)0)->no_cnts);
}
bytes = sizeof(((struct s390_ctrset_read *)0)->no_cpus) + nr_cpu_ids *
(bytes + sizeof(((struct s390_ctrset_cpudata *)0)->cpu_nr) +
sizeof(((struct s390_ctrset_cpudata *)0)->no_sets));
debug_sprintf_event(cf_diag_dbg, 5, "%s bytes %ld\n", __func__,
bytes);
put_cpu_ptr(&cpu_cf_events);
return bytes;
}
static long cf_diag_ioctl_read(unsigned long arg)
{
struct s390_ctrset_read read;
int ret = 0;
debug_sprintf_event(cf_diag_dbg, 5, "%s\n", __func__);
if (copy_from_user(&read, (char __user *)arg, sizeof(read)))
return -EFAULT;
ret = cf_diag_all_read(arg);
debug_sprintf_event(cf_diag_dbg, 5, "%s ret %d\n", __func__, ret);
return ret;
}
static long cf_diag_ioctl_stop(void)
{
int ret;
debug_sprintf_event(cf_diag_dbg, 5, "%s\n", __func__);
ret = cf_diag_all_stop();
cf_diag_ctrset_clear();
debug_sprintf_event(cf_diag_dbg, 5, "%s ret %d\n", __func__, ret);
return ret;
}
static long cf_diag_ioctl_start(unsigned long arg)
{
struct s390_ctrset_start __user *ustart;
struct s390_ctrset_start start;
void __user *umask;
unsigned int len;
int ret = 0;
size_t need;
if (cf_diag_ctrset.ctrset)
return -EBUSY;
ustart = (struct s390_ctrset_start __user *)arg;
if (copy_from_user(&start, ustart, sizeof(start)))
return -EFAULT;
if (start.version != S390_HWCTR_START_VERSION)
return -EINVAL;
if (start.counter_sets & ~(cpumf_ctr_ctl[CPUMF_CTR_SET_BASIC] |
cpumf_ctr_ctl[CPUMF_CTR_SET_USER] |
cpumf_ctr_ctl[CPUMF_CTR_SET_CRYPTO] |
cpumf_ctr_ctl[CPUMF_CTR_SET_EXT] |
cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG]))
return -EINVAL; /* Invalid counter set */
if (!start.counter_sets)
return -EINVAL; /* No counter set at all? */
cpumask_clear(&cf_diag_ctrset.mask);
len = min_t(u64, start.cpumask_len, cpumask_size());
umask = (void __user *)start.cpumask;
if (copy_from_user(&cf_diag_ctrset.mask, umask, len))
return -EFAULT;
if (cpumask_empty(&cf_diag_ctrset.mask))
return -EINVAL;
need = cf_diag_needspace(start.counter_sets);
if (put_user(need, &ustart->data_bytes))
ret = -EFAULT;
if (ret)
goto out;
cf_diag_ctrset.ctrset = start.counter_sets;
ret = cf_diag_all_start();
out:
if (ret)
cf_diag_ctrset_clear();
debug_sprintf_event(cf_diag_dbg, 2, "%s sets %#lx need %ld ret %d\n",
__func__, cf_diag_ctrset.ctrset, need, ret);
return ret;
}
static long cf_diag_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
int ret;
debug_sprintf_event(cf_diag_dbg, 2, "%s cmd %#x arg %lx\n", __func__,
cmd, arg);
get_online_cpus();
mutex_lock(&cf_diag_ctrset_mutex);
switch (cmd) {
case S390_HWCTR_START:
ret = cf_diag_ioctl_start(arg);
break;
case S390_HWCTR_STOP:
ret = cf_diag_ioctl_stop();
break;
case S390_HWCTR_READ:
ret = cf_diag_ioctl_read(arg);
break;
default:
ret = -ENOTTY;
break;
}
mutex_unlock(&cf_diag_ctrset_mutex);
put_online_cpus();
debug_sprintf_event(cf_diag_dbg, 2, "%s ret %d\n", __func__, ret);
return ret;
}
static const struct file_operations cf_diag_fops = {
.owner = THIS_MODULE,
.open = cf_diag_open,
.release = cf_diag_release,
.unlocked_ioctl = cf_diag_ioctl,
.compat_ioctl = cf_diag_ioctl,
.llseek = no_llseek
};
static struct miscdevice cf_diag_dev = {
.name = S390_HWCTR_DEVICE,
.minor = MISC_DYNAMIC_MINOR,
.fops = &cf_diag_fops,
};
static int cf_diag_online_cpu(unsigned int cpu)
{
struct cf_diag_call_on_cpu_parm p;
mutex_lock(&cf_diag_ctrset_mutex);
if (!cf_diag_ctrset.ctrset)
goto out;
p.sets = cf_diag_ctrset.ctrset;
cf_diag_ioctl_on(&p);
out:
mutex_unlock(&cf_diag_ctrset_mutex);
return 0;
}
static int cf_diag_offline_cpu(unsigned int cpu)
{
struct cf_diag_call_on_cpu_parm p;
mutex_lock(&cf_diag_ctrset_mutex);
if (!cf_diag_ctrset.ctrset)
goto out;
p.sets = cf_diag_ctrset.ctrset;
cf_diag_ioctl_off(&p);
out:
mutex_unlock(&cf_diag_ctrset_mutex);
return 0;
}
/* Initialize the counter set PMU to generate complete counter set data as
* event raw data. This relies on the CPU Measurement Counter Facility device
* already being loaded and initialized.
*/
static int __init cf_diag_init(void)
{
struct cpumf_ctr_info info;
size_t need;
int rc;
if (!kernel_cpumcf_avail() || !stccm_avail() || qctri(&info))
return -ENODEV;
cf_diag_get_cpu_speed();
/* Make sure the counter set data fits into predefined buffer. */
need = cf_diag_ctrset_maxsize(&info);
if (need > sizeof(((struct cf_diag_csd *)0)->start)) {
pr_err("Insufficient memory for PMU(cpum_cf_diag) need=%zu\n",
need);
return -ENOMEM;
}
rc = misc_register(&cf_diag_dev);
if (rc) {
pr_err("Registration of /dev/" S390_HWCTR_DEVICE
"failed rc=%d\n", rc);
goto out;
}
/* Setup s390dbf facility */
cf_diag_dbg = debug_register(KMSG_COMPONENT, 2, 1, 128);
if (!cf_diag_dbg) {
pr_err("Registration of s390dbf(cpum_cf_diag) failed\n");
rc = -ENOMEM;
goto out_dbf;
}
debug_register_view(cf_diag_dbg, &debug_sprintf_view);
rc = perf_pmu_register(&cf_diag, "cpum_cf_diag", -1);
if (rc) {
pr_err("Registration of PMU(cpum_cf_diag) failed with rc=%i\n",
rc);
goto out_perf;
}
rc = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_S390_CFD_ONLINE,
"perf/s390/cfd:online",
cf_diag_online_cpu, cf_diag_offline_cpu);
if (!rc)
goto out;
pr_err("Registration of CPUHP_AP_PERF_S390_CFD_ONLINE failed rc=%i\n",
rc);
perf_pmu_unregister(&cf_diag);
out_perf:
debug_unregister_view(cf_diag_dbg, &debug_sprintf_view);
debug_unregister(cf_diag_dbg);
out_dbf:
misc_deregister(&cf_diag_dev);
out:
return rc;
}
device_initcall(cf_diag_init);
include/linux/cpuhotplug.h
View file @ a029a4ea
......@@ -171,7 +171,6 @@ enum cpuhp_state {
CPUHP_AP_PERF_X86_CSTATE_ONLINE,
CPUHP_AP_PERF_X86_IDXD_ONLINE,
CPUHP_AP_PERF_S390_CF_ONLINE,
CPUHP_AP_PERF_S390_CFD_ONLINE,
CPUHP_AP_PERF_S390_SF_ONLINE,
CPUHP_AP_PERF_ARM_CCI_ONLINE,
CPUHP_AP_PERF_ARM_CCN_ONLINE,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7