Commit f92b5462 authored by James Morse's avatar James Morse Committed by Catalin Marinas

firmware: arm_sdei: add support for CPU private events

Private SDE events are per-cpu, and need to be registered and enabled
on each CPU.

Hide this detail from the caller by adapting our {,un}register and
{en,dis}able calls to send an IPI to each CPU if the event is private.

CPU private events are unregistered when the CPU is powered-off, and
re-registered when the CPU is brought back online. This saves bringing
secondary cores back online to call private_reset() on shutdown, kexec
and resume from hibernate.
Signed-off-by: default avatarJames Morse <james.morse@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent da351827
...@@ -5,9 +5,11 @@ ...@@ -5,9 +5,11 @@
#include <linux/acpi.h> #include <linux/acpi.h>
#include <linux/arm_sdei.h> #include <linux/arm_sdei.h>
#include <linux/arm-smccc.h> #include <linux/arm-smccc.h>
#include <linux/atomic.h>
#include <linux/bitops.h> #include <linux/bitops.h>
#include <linux/compiler.h> #include <linux/compiler.h>
#include <linux/cpuhotplug.h> #include <linux/cpuhotplug.h>
#include <linux/cpu.h>
#include <linux/cpu_pm.h> #include <linux/cpu_pm.h>
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/hardirq.h> #include <linux/hardirq.h>
...@@ -52,7 +54,13 @@ struct sdei_event { ...@@ -52,7 +54,13 @@ struct sdei_event {
u8 priority; u8 priority;
/* This pointer is handed to firmware as the event argument. */ /* This pointer is handed to firmware as the event argument. */
struct sdei_registered_event *registered; union {
/* Shared events */
struct sdei_registered_event *registered;
/* CPU private events */
struct sdei_registered_event __percpu *private_registered;
};
}; };
/* Take the mutex for any API call or modification. Take the mutex first. */ /* Take the mutex for any API call or modification. Take the mutex first. */
...@@ -62,6 +70,34 @@ static DEFINE_MUTEX(sdei_events_lock); ...@@ -62,6 +70,34 @@ static DEFINE_MUTEX(sdei_events_lock);
static DEFINE_SPINLOCK(sdei_list_lock); static DEFINE_SPINLOCK(sdei_list_lock);
static LIST_HEAD(sdei_list); static LIST_HEAD(sdei_list);
/* Private events are registered/enabled via IPI passing one of these */
struct sdei_crosscall_args {
struct sdei_event *event;
atomic_t errors;
int first_error;
};
#define CROSSCALL_INIT(arg, event) (arg.event = event, \
arg.first_error = 0, \
atomic_set(&arg.errors, 0))
static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
{
struct sdei_crosscall_args arg;
CROSSCALL_INIT(arg, event);
on_each_cpu(fn, &arg, true);
return arg.first_error;
}
static inline void
sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
{
if (err && (atomic_inc_return(&arg->errors) == 1))
arg->first_error = err;
}
static int sdei_to_linux_errno(unsigned long sdei_err) static int sdei_to_linux_errno(unsigned long sdei_err)
{ {
switch (sdei_err) { switch (sdei_err) {
...@@ -207,6 +243,26 @@ static struct sdei_event *sdei_event_create(u32 event_num, ...@@ -207,6 +243,26 @@ static struct sdei_event *sdei_event_create(u32 event_num,
reg->callback = cb; reg->callback = cb;
reg->callback_arg = cb_arg; reg->callback_arg = cb_arg;
event->registered = reg; event->registered = reg;
} else {
int cpu;
struct sdei_registered_event __percpu *regs;
regs = alloc_percpu(struct sdei_registered_event);
if (!regs) {
kfree(event);
return ERR_PTR(-ENOMEM);
}
for_each_possible_cpu(cpu) {
reg = per_cpu_ptr(regs, cpu);
reg->event_num = event->event_num;
reg->priority = event->priority;
reg->callback = cb;
reg->callback_arg = cb_arg;
}
event->private_registered = regs;
} }
if (sdei_event_find(event_num)) { if (sdei_event_find(event_num)) {
...@@ -232,6 +288,8 @@ static void sdei_event_destroy(struct sdei_event *event) ...@@ -232,6 +288,8 @@ static void sdei_event_destroy(struct sdei_event *event)
if (event->type == SDEI_EVENT_TYPE_SHARED) if (event->type == SDEI_EVENT_TYPE_SHARED)
kfree(event->registered); kfree(event->registered);
else
free_percpu(event->private_registered);
kfree(event); kfree(event);
} }
...@@ -262,11 +320,6 @@ static void _ipi_mask_cpu(void *ignored) ...@@ -262,11 +320,6 @@ static void _ipi_mask_cpu(void *ignored)
sdei_mask_local_cpu(); sdei_mask_local_cpu();
} }
static int sdei_cpuhp_down(unsigned int ignored)
{
return sdei_mask_local_cpu();
}
int sdei_unmask_local_cpu(void) int sdei_unmask_local_cpu(void)
{ {
int err; int err;
...@@ -288,11 +341,6 @@ static void _ipi_unmask_cpu(void *ignored) ...@@ -288,11 +341,6 @@ static void _ipi_unmask_cpu(void *ignored)
sdei_unmask_local_cpu(); sdei_unmask_local_cpu();
} }
static int sdei_cpuhp_up(unsigned int ignored)
{
return sdei_unmask_local_cpu();
}
static void _ipi_private_reset(void *ignored) static void _ipi_private_reset(void *ignored)
{ {
int err; int err;
...@@ -337,6 +385,19 @@ static int sdei_api_event_enable(u32 event_num) ...@@ -337,6 +385,19 @@ static int sdei_api_event_enable(u32 event_num)
0, NULL); 0, NULL);
} }
/* Called directly by the hotplug callbacks */
static void _local_event_enable(void *data)
{
int err;
struct sdei_crosscall_args *arg = data;
WARN_ON_ONCE(preemptible());
err = sdei_api_event_enable(arg->event->event_num);
sdei_cross_call_return(arg, err);
}
int sdei_event_enable(u32 event_num) int sdei_event_enable(u32 event_num)
{ {
int err = -EINVAL; int err = -EINVAL;
...@@ -355,6 +416,8 @@ int sdei_event_enable(u32 event_num) ...@@ -355,6 +416,8 @@ int sdei_event_enable(u32 event_num)
if (event->type == SDEI_EVENT_TYPE_SHARED) if (event->type == SDEI_EVENT_TYPE_SHARED)
err = sdei_api_event_enable(event->event_num); err = sdei_api_event_enable(event->event_num);
else
err = sdei_do_cross_call(_local_event_enable, event);
mutex_unlock(&sdei_events_lock); mutex_unlock(&sdei_events_lock);
return err; return err;
...@@ -367,6 +430,16 @@ static int sdei_api_event_disable(u32 event_num) ...@@ -367,6 +430,16 @@ static int sdei_api_event_disable(u32 event_num)
0, 0, NULL); 0, 0, NULL);
} }
static void _ipi_event_disable(void *data)
{
int err;
struct sdei_crosscall_args *arg = data;
err = sdei_api_event_disable(arg->event->event_num);
sdei_cross_call_return(arg, err);
}
int sdei_event_disable(u32 event_num) int sdei_event_disable(u32 event_num)
{ {
int err = -EINVAL; int err = -EINVAL;
...@@ -385,6 +458,8 @@ int sdei_event_disable(u32 event_num) ...@@ -385,6 +458,8 @@ int sdei_event_disable(u32 event_num)
if (event->type == SDEI_EVENT_TYPE_SHARED) if (event->type == SDEI_EVENT_TYPE_SHARED)
err = sdei_api_event_disable(event->event_num); err = sdei_api_event_disable(event->event_num);
else
err = sdei_do_cross_call(_ipi_event_disable, event);
mutex_unlock(&sdei_events_lock); mutex_unlock(&sdei_events_lock);
return err; return err;
...@@ -397,6 +472,19 @@ static int sdei_api_event_unregister(u32 event_num) ...@@ -397,6 +472,19 @@ static int sdei_api_event_unregister(u32 event_num)
0, 0, 0, NULL); 0, 0, 0, NULL);
} }
/* Called directly by the hotplug callbacks */
static void _local_event_unregister(void *data)
{
int err;
struct sdei_crosscall_args *arg = data;
WARN_ON_ONCE(preemptible());
err = sdei_api_event_unregister(arg->event->event_num);
sdei_cross_call_return(arg, err);
}
static int _sdei_event_unregister(struct sdei_event *event) static int _sdei_event_unregister(struct sdei_event *event)
{ {
lockdep_assert_held(&sdei_events_lock); lockdep_assert_held(&sdei_events_lock);
...@@ -409,7 +497,7 @@ static int _sdei_event_unregister(struct sdei_event *event) ...@@ -409,7 +497,7 @@ static int _sdei_event_unregister(struct sdei_event *event)
if (event->type == SDEI_EVENT_TYPE_SHARED) if (event->type == SDEI_EVENT_TYPE_SHARED)
return sdei_api_event_unregister(event->event_num); return sdei_api_event_unregister(event->event_num);
return -EINVAL; return sdei_do_cross_call(_local_event_unregister, event);
} }
int sdei_event_unregister(u32 event_num) int sdei_event_unregister(u32 event_num)
...@@ -473,17 +561,50 @@ static int sdei_api_event_register(u32 event_num, unsigned long entry_point, ...@@ -473,17 +561,50 @@ static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
flags, affinity, NULL); flags, affinity, NULL);
} }
/* Called directly by the hotplug callbacks */
static void _local_event_register(void *data)
{
int err;
struct sdei_registered_event *reg;
struct sdei_crosscall_args *arg = data;
WARN_ON(preemptible());
reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
reg, 0, 0);
sdei_cross_call_return(arg, err);
}
static int _sdei_event_register(struct sdei_event *event) static int _sdei_event_register(struct sdei_event *event)
{ {
int err;
lockdep_assert_held(&sdei_events_lock); lockdep_assert_held(&sdei_events_lock);
spin_lock(&sdei_list_lock);
event->reregister = true;
spin_unlock(&sdei_list_lock);
if (event->type == SDEI_EVENT_TYPE_SHARED) if (event->type == SDEI_EVENT_TYPE_SHARED)
return sdei_api_event_register(event->event_num, return sdei_api_event_register(event->event_num,
sdei_entry_point, sdei_entry_point,
event->registered, event->registered,
SDEI_EVENT_REGISTER_RM_ANY, 0); SDEI_EVENT_REGISTER_RM_ANY, 0);
return -EINVAL;
err = sdei_do_cross_call(_local_event_register, event);
if (err) {
spin_lock(&sdei_list_lock);
event->reregister = false;
event->reenable = false;
spin_unlock(&sdei_list_lock);
sdei_do_cross_call(_local_event_unregister, event);
}
return err;
} }
int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg) int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
...@@ -538,6 +659,8 @@ static int sdei_reregister_event(struct sdei_event *event) ...@@ -538,6 +659,8 @@ static int sdei_reregister_event(struct sdei_event *event)
if (event->reenable) { if (event->reenable) {
if (event->type == SDEI_EVENT_TYPE_SHARED) if (event->type == SDEI_EVENT_TYPE_SHARED)
err = sdei_api_event_enable(event->event_num); err = sdei_api_event_enable(event->event_num);
else
err = sdei_do_cross_call(_local_event_enable, event);
} }
if (err) if (err)
...@@ -569,6 +692,62 @@ static int sdei_reregister_shared(void) ...@@ -569,6 +692,62 @@ static int sdei_reregister_shared(void)
return err; return err;
} }
static int sdei_cpuhp_down(unsigned int cpu)
{
struct sdei_event *event;
struct sdei_crosscall_args arg;
/* un-register private events */
spin_lock(&sdei_list_lock);
list_for_each_entry(event, &sdei_list, list) {
if (event->type == SDEI_EVENT_TYPE_SHARED)
continue;
CROSSCALL_INIT(arg, event);
/* call the cross-call function locally... */
_local_event_unregister(&arg);
if (arg.first_error)
pr_err("Failed to unregister event %u: %d\n",
event->event_num, arg.first_error);
}
spin_unlock(&sdei_list_lock);
return sdei_mask_local_cpu();
}
static int sdei_cpuhp_up(unsigned int cpu)
{
struct sdei_event *event;
struct sdei_crosscall_args arg;
/* re-register/enable private events */
spin_lock(&sdei_list_lock);
list_for_each_entry(event, &sdei_list, list) {
if (event->type == SDEI_EVENT_TYPE_SHARED)
continue;
if (event->reregister) {
CROSSCALL_INIT(arg, event);
/* call the cross-call function locally... */
_local_event_register(&arg);
if (arg.first_error)
pr_err("Failed to re-register event %u: %d\n",
event->event_num, arg.first_error);
}
if (event->reenable) {
CROSSCALL_INIT(arg, event);
_local_event_enable(&arg);
if (arg.first_error)
pr_err("Failed to re-enable event %u: %d\n",
event->event_num, arg.first_error);
}
}
spin_unlock(&sdei_list_lock);
return sdei_unmask_local_cpu();
}
/* When entering idle, mask/unmask events for this cpu */ /* When entering idle, mask/unmask events for this cpu */
static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action, static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
void *data) void *data)
...@@ -622,6 +801,7 @@ static int sdei_device_freeze(struct device *dev) ...@@ -622,6 +801,7 @@ static int sdei_device_freeze(struct device *dev)
{ {
int err; int err;
/* unregister private events */
cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING); cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
err = sdei_unregister_shared(); err = sdei_unregister_shared();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment