Commit 5110459f authored by Arnd Bergmann's avatar Arnd Bergmann Committed by Paul Mackerras

[PATCH] spufs: Improved SPU preemptability.

This patch makes it easier to preempt an SPU context by
having the scheduler hold ctx->state_sema for much shorter
periods of time.

As part of this restructuring, the control logic for the "run"
operation is moved from arch/ppc64/kernel/spu_base.c to
fs/spufs/file.c.  Of course the base retains "bottom half"
handlers for class{0,1} irqs.  The new run loop will re-acquire
an SPU if preempted.

From: Mark Nutter <mnutter@us.ibm.com>
Signed-off-by: default avatarArnd Bergmann <arndb@de.ibm.com>
Signed-off-by: default avatarPaul Mackerras <paulus@samba.org>
parent 3b3d22cb
......@@ -130,7 +130,8 @@ static int __spu_trap_data_map(struct spu *spu, unsigned long ea, u64 dsisr)
spu->dar = ea;
spu->dsisr = dsisr;
mb();
wake_up(&spu->stop_wq);
if (spu->stop_callback)
spu->stop_callback(spu);
return 0;
}
......@@ -151,7 +152,8 @@ static int __spu_trap_stop(struct spu *spu)
{
pr_debug("%s\n", __FUNCTION__);
spu->stop_code = in_be32(&spu->problem->spu_status_R);
wake_up(&spu->stop_wq);
if (spu->stop_callback)
spu->stop_callback(spu);
return 0;
}
......@@ -159,7 +161,8 @@ static int __spu_trap_halt(struct spu *spu)
{
pr_debug("%s\n", __FUNCTION__);
spu->stop_code = in_be32(&spu->problem->spu_status_R);
wake_up(&spu->stop_wq);
if (spu->stop_callback)
spu->stop_callback(spu);
return 0;
}
......@@ -190,12 +193,13 @@ spu_irq_class_0(int irq, void *data, struct pt_regs *regs)
spu = data;
spu->class_0_pending = 1;
wake_up(&spu->stop_wq);
if (spu->stop_callback)
spu->stop_callback(spu);
return IRQ_HANDLED;
}
static int
int
spu_irq_class_0_bottom(struct spu *spu)
{
unsigned long stat;
......@@ -214,8 +218,10 @@ spu_irq_class_0_bottom(struct spu *spu)
__spu_trap_error(spu);
out_be64(&spu->priv1->int_stat_class0_RW, stat);
return 0;
return (stat & 0x7) ? -EIO : 0;
}
EXPORT_SYMBOL_GPL(spu_irq_class_0_bottom);
static irqreturn_t
spu_irq_class_1(int irq, void *data, struct pt_regs *regs)
......@@ -250,6 +256,7 @@ spu_irq_class_1(int irq, void *data, struct pt_regs *regs)
return stat ? IRQ_HANDLED : IRQ_NONE;
}
EXPORT_SYMBOL_GPL(spu_irq_class_1_bottom);
static irqreturn_t
spu_irq_class_2(int irq, void *data, struct pt_regs *regs)
......@@ -478,7 +485,7 @@ static int spu_handle_mm_fault(struct spu *spu)
return -EFAULT;
}
static int spu_handle_pte_fault(struct spu *spu)
int spu_irq_class_1_bottom(struct spu *spu)
{
u64 ea, dsisr, access, error = 0UL;
int ret = 0;
......@@ -508,76 +515,6 @@ static int spu_handle_pte_fault(struct spu *spu)
return ret;
}
static inline int spu_pending(struct spu *spu, u32 * stat)
{
struct spu_problem __iomem *prob = spu->problem;
u64 pte_fault;
*stat = in_be32(&prob->spu_status_R);
pte_fault = spu->dsisr &
(MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED);
return (!(*stat & 0x1) || pte_fault || spu->class_0_pending) ? 1 : 0;
}
int spu_run(struct spu *spu)
{
struct spu_problem __iomem *prob;
struct spu_priv1 __iomem *priv1;
struct spu_priv2 __iomem *priv2;
u32 status;
int ret;
prob = spu->problem;
priv1 = spu->priv1;
priv2 = spu->priv2;
/* Let SPU run. */
eieio();
out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
do {
ret = wait_event_interruptible(spu->stop_wq,
spu_pending(spu, &status));
if (spu->dsisr &
(MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED))
ret = spu_handle_pte_fault(spu);
if (spu->class_0_pending)
spu_irq_class_0_bottom(spu);
if (!ret && signal_pending(current))
ret = -ERESTARTSYS;
} while (!ret && !(status &
(SPU_STATUS_STOPPED_BY_STOP |
SPU_STATUS_STOPPED_BY_HALT)));
/* Ensure SPU is stopped. */
out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
eieio();
while (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)
cpu_relax();
out_be64(&priv2->slb_invalidate_all_W, 0);
out_be64(&priv1->tlb_invalidate_entry_W, 0UL);
eieio();
/* Check for SPU breakpoint. */
if (unlikely(current->ptrace & PT_PTRACED)) {
status = in_be32(&prob->spu_status_R);
if ((status & SPU_STATUS_STOPPED_BY_STOP)
&& status >> SPU_STOP_STATUS_SHIFT == 0x3fff) {
force_sig(SIGTRAP, current);
ret = -ERESTARTSYS;
}
}
return ret;
}
EXPORT_SYMBOL_GPL(spu_run);
static void __iomem * __init map_spe_prop(struct device_node *n,
const char *name)
{
......@@ -693,9 +630,9 @@ static int __init create_spu(struct device_node *spe)
out_be64(&spu->priv1->mfc_sdr_RW, mfspr(SPRN_SDR1));
out_be64(&spu->priv1->mfc_sr1_RW, 0x33);
init_waitqueue_head(&spu->stop_wq);
spu->ibox_callback = NULL;
spu->wbox_callback = NULL;
spu->stop_callback = NULL;
down(&spu_mutex);
spu->number = number++;
......
......@@ -232,6 +232,23 @@ static char *spu_backing_get_ls(struct spu_context *ctx)
return ctx->csa.lscsa->ls;
}
static void spu_backing_runcntl_write(struct spu_context *ctx, u32 val)
{
spin_lock(&ctx->csa.register_lock);
ctx->csa.prob.spu_runcntl_RW = val;
if (val & SPU_RUNCNTL_RUNNABLE) {
ctx->csa.prob.spu_status_R |= SPU_STATUS_RUNNING;
} else {
ctx->csa.prob.spu_status_R &= ~SPU_STATUS_RUNNING;
}
spin_unlock(&ctx->csa.register_lock);
}
static void spu_backing_runcntl_stop(struct spu_context *ctx)
{
spu_backing_runcntl_write(ctx, SPU_RUNCNTL_STOP);
}
struct spu_context_ops spu_backing_ops = {
.mbox_read = spu_backing_mbox_read,
.mbox_stat_read = spu_backing_mbox_stat_read,
......@@ -249,4 +266,6 @@ struct spu_context_ops spu_backing_ops = {
.npc_write = spu_backing_npc_write,
.status_read = spu_backing_status_read,
.get_ls = spu_backing_get_ls,
.runcntl_write = spu_backing_runcntl_write,
.runcntl_stop = spu_backing_runcntl_stop,
};
......@@ -45,6 +45,7 @@ struct spu_context *alloc_spu_context(struct address_space *local_store)
init_rwsem(&ctx->state_sema);
init_waitqueue_head(&ctx->ibox_wq);
init_waitqueue_head(&ctx->wbox_wq);
init_waitqueue_head(&ctx->stop_wq);
ctx->ibox_fasync = NULL;
ctx->wbox_fasync = NULL;
ctx->state = SPU_STATE_SAVED;
......@@ -105,7 +106,7 @@ void spu_release(struct spu_context *ctx)
up_read(&ctx->state_sema);
}
static void spu_unmap_mappings(struct spu_context *ctx)
void spu_unmap_mappings(struct spu_context *ctx)
{
unmap_mapping_range(ctx->local_store, 0, LS_SIZE, 1);
}
......@@ -126,7 +127,6 @@ int spu_acquire_runnable(struct spu_context *ctx)
down_write(&ctx->state_sema);
if (ctx->state == SPU_STATE_SAVED) {
spu_unmap_mappings(ctx);
ret = spu_activate(ctx, 0);
ctx->state = SPU_STATE_RUNNABLE;
}
......@@ -154,7 +154,6 @@ void spu_acquire_saved(struct spu_context *ctx)
down_write(&ctx->state_sema);
if (ctx->state == SPU_STATE_RUNNABLE) {
spu_unmap_mappings(ctx);
spu_deactivate(ctx);
ctx->state = SPU_STATE_SAVED;
}
......
......@@ -25,6 +25,7 @@
#include <linux/module.h>
#include <linux/pagemap.h>
#include <linux/poll.h>
#include <linux/ptrace.h>
#include <asm/io.h>
#include <asm/semaphore.h>
......@@ -540,26 +541,122 @@ static struct file_operations spufs_wbox_stat_fops = {
.read = spufs_wbox_stat_read,
};
long spufs_run_spu(struct file *file, struct spu_context *ctx,
u32 *npc, u32 *status)
/* interrupt-level stop callback function. */
void spufs_stop_callback(struct spu *spu)
{
struct spu_context *ctx = spu->ctx;
wake_up_all(&ctx->stop_wq);
}
static inline int spu_stopped(struct spu_context *ctx, u32 * stat)
{
struct spu *spu;
u64 pte_fault;
*stat = ctx->ops->status_read(ctx);
if (ctx->state != SPU_STATE_RUNNABLE)
return 1;
spu = ctx->spu;
pte_fault = spu->dsisr &
(MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED);
return (!(*stat & 0x1) || pte_fault || spu->class_0_pending) ? 1 : 0;
}
static inline int spu_run_init(struct spu_context *ctx, u32 * npc,
u32 * status)
{
int ret;
ret = spu_acquire_runnable(ctx);
if (ret)
if ((ret = spu_acquire_runnable(ctx)) != 0)
return ret;
ctx->ops->npc_write(ctx, *npc);
ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_RUNNABLE);
return 0;
}
ret = spu_run(ctx->spu);
if (!ret)
ret = ctx->ops->status_read(ctx);
static inline int spu_run_fini(struct spu_context *ctx, u32 * npc,
u32 * status)
{
int ret = 0;
*status = ctx->ops->status_read(ctx);
*npc = ctx->ops->npc_read(ctx);
spu_release(ctx);
if (signal_pending(current))
ret = -ERESTARTSYS;
if (unlikely(current->ptrace & PT_PTRACED)) {
if ((*status & SPU_STATUS_STOPPED_BY_STOP)
&& (*status >> SPU_STOP_STATUS_SHIFT) == 0x3fff) {
force_sig(SIGTRAP, current);
ret = -ERESTARTSYS;
}
}
return ret;
}
static inline int spu_reacquire_runnable(struct spu_context *ctx, u32 *npc,
u32 *status)
{
int ret;
if ((ret = spu_run_fini(ctx, npc, status)) != 0)
return ret;
if (*status & (SPU_STATUS_STOPPED_BY_STOP |
SPU_STATUS_STOPPED_BY_HALT)) {
return *status;
}
if ((ret = spu_run_init(ctx, npc, status)) != 0)
return ret;
return 0;
}
static inline int spu_process_events(struct spu_context *ctx)
{
struct spu *spu = ctx->spu;
u64 pte_fault = MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED;
int ret = 0;
if (spu->dsisr & pte_fault)
ret = spu_irq_class_1_bottom(spu);
if (spu->class_0_pending)
ret = spu_irq_class_0_bottom(spu);
if (!ret && signal_pending(current))
ret = -ERESTARTSYS;
return ret;
}
long spufs_run_spu(struct file *file, struct spu_context *ctx,
u32 * npc, u32 * status)
{
int ret;
if ((ret = spu_run_init(ctx, npc, status)) != 0)
return ret;
do {
ret = spufs_wait(ctx->stop_wq, spu_stopped(ctx, status));
if (unlikely(ret))
break;
if (unlikely(ctx->state != SPU_STATE_RUNNABLE)) {
ret = spu_reacquire_runnable(ctx, npc, status);
if (ret) {
return ret;
}
continue;
}
ret = spu_process_events(ctx);
} while (!ret && !(*status & (SPU_STATUS_STOPPED_BY_STOP |
SPU_STATUS_STOPPED_BY_HALT)));
ctx->ops->runcntl_stop(ctx);
ret = spu_run_fini(ctx, npc, status);
if (!ret)
ret = *status;
spu_yield(ctx);
return ret;
}
......
......@@ -186,6 +186,21 @@ static char *spu_hw_get_ls(struct spu_context *ctx)
return ctx->spu->local_store;
}
static void spu_hw_runcntl_write(struct spu_context *ctx, u32 val)
{
eieio();
out_be32(&ctx->spu->problem->spu_runcntl_RW, val);
}
static void spu_hw_runcntl_stop(struct spu_context *ctx)
{
spin_lock_irq(&ctx->spu->register_lock);
out_be32(&ctx->spu->problem->spu_runcntl_RW, SPU_RUNCNTL_STOP);
while (in_be32(&ctx->spu->problem->spu_status_R) & SPU_STATUS_RUNNING)
cpu_relax();
spin_unlock_irq(&ctx->spu->register_lock);
}
struct spu_context_ops spu_hw_ops = {
.mbox_read = spu_hw_mbox_read,
.mbox_stat_read = spu_hw_mbox_stat_read,
......@@ -203,4 +218,6 @@ struct spu_context_ops spu_hw_ops = {
.npc_write = spu_hw_npc_write,
.status_read = spu_hw_status_read,
.get_ls = spu_hw_get_ls,
.runcntl_write = spu_hw_runcntl_write,
.runcntl_stop = spu_hw_runcntl_stop,
};
......@@ -119,7 +119,8 @@ static void prio_wakeup(struct spu_runqueue *rq)
}
}
static void prio_wait(struct spu_runqueue *rq, u64 flags)
static void prio_wait(struct spu_runqueue *rq, struct spu_context *ctx,
u64 flags)
{
int prio = current->prio;
wait_queue_head_t *wq = &rq->prio.waitq[prio];
......@@ -130,9 +131,11 @@ static void prio_wait(struct spu_runqueue *rq, u64 flags)
prepare_to_wait_exclusive(wq, &wait, TASK_INTERRUPTIBLE);
if (!signal_pending(current)) {
up(&rq->sem);
up_write(&ctx->state_sema);
pr_debug("%s: pid=%d prio=%d\n", __FUNCTION__,
current->pid, current->prio);
schedule();
down_write(&ctx->state_sema);
down(&rq->sem);
}
finish_wait(wq, &wait);
......@@ -173,7 +176,9 @@ static inline void bind_context(struct spu *spu, struct spu_context *ctx)
mm_needs_global_tlbie(spu->mm);
spu->ibox_callback = spufs_ibox_callback;
spu->wbox_callback = spufs_wbox_callback;
spu->stop_callback = spufs_stop_callback;
mb();
spu_unmap_mappings(ctx);
spu_restore(&ctx->csa, spu);
}
......@@ -181,10 +186,12 @@ static inline void unbind_context(struct spu *spu, struct spu_context *ctx)
{
pr_debug("%s: unbind pid=%d SPU=%d\n", __FUNCTION__,
spu->pid, spu->number);
spu_unmap_mappings(ctx);
spu_save(&ctx->csa, spu);
ctx->state = SPU_STATE_SAVED;
spu->ibox_callback = NULL;
spu->wbox_callback = NULL;
spu->stop_callback = NULL;
spu->mm = NULL;
spu->pid = 0;
spu->prio = MAX_PRIO;
......@@ -196,37 +203,35 @@ static inline void unbind_context(struct spu *spu, struct spu_context *ctx)
static struct spu *preempt_active(struct spu_runqueue *rq)
{
struct list_head *p;
struct spu_context *ctx;
struct spu *spu;
struct spu *worst, *spu;
/* Future: implement real preemption. For now just
* boot a lower priority ctx that is in "detached"
* state, i.e. on a processor but not currently in
* spu_run().
*/
worst = list_entry(rq->active_list.next, struct spu, sched_list);
list_for_each(p, &rq->active_list) {
spu = list_entry(p, struct spu, sched_list);
if (current->prio < spu->prio) {
ctx = spu->ctx;
if (down_write_trylock(&ctx->state_sema)) {
if (ctx->state != SPU_STATE_RUNNABLE) {
up_write(&ctx->state_sema);
continue;
}
pr_debug("%s: booting pid=%d from SPU %d\n",
__FUNCTION__, spu->pid, spu->number);
del_active(rq, spu);
up(&rq->sem);
unbind_context(spu, ctx);
up_write(&ctx->state_sema);
return spu;
}
if (spu->prio > worst->prio) {
worst = spu;
}
}
if (current->prio < worst->prio) {
struct spu_context *ctx = worst->ctx;
spu = worst;
if (down_write_trylock(&ctx->state_sema)) {
pr_debug("%s: booting pid=%d from SPU %d\n",
__FUNCTION__, spu->pid, spu->number);
del_active(rq, spu);
up(&rq->sem);
wake_up_all(&ctx->stop_wq);
ctx->ops->runcntl_stop(ctx);
unbind_context(spu, ctx);
up_write(&ctx->state_sema);
return spu;
}
}
return NULL;
}
static struct spu *get_idle_spu(u64 flags)
static struct spu *get_idle_spu(struct spu_context *ctx, u64 flags)
{
struct spu_runqueue *rq;
struct spu *spu = NULL;
......@@ -255,7 +260,7 @@ static struct spu *get_idle_spu(u64 flags)
if ((spu = preempt_active(rq)) != NULL)
return spu;
}
prio_wait(rq, flags);
prio_wait(rq, ctx, flags);
if (signal_pending(current)) {
prio_wakeup(rq);
spu = NULL;
......@@ -322,7 +327,7 @@ int spu_activate(struct spu_context *ctx, u64 flags)
if (ctx->spu)
return 0;
spu = get_idle_spu(flags);
spu = get_idle_spu(ctx, flags);
if (!spu)
return (signal_pending(current)) ? -ERESTARTSYS : -EAGAIN;
bind_context(spu, ctx);
......@@ -347,17 +352,19 @@ void spu_deactivate(struct spu_context *ctx)
void spu_yield(struct spu_context *ctx)
{
struct spu *spu;
int need_yield = 0;
if (!down_write_trylock(&ctx->state_sema))
return;
down_write(&ctx->state_sema);
spu = ctx->spu;
if ((ctx->state == SPU_STATE_RUNNABLE) &&
(sched_find_first_bit(spu->rq->prio.bitmap) <= current->prio)) {
if (spu && (sched_find_first_bit(spu->rq->prio.bitmap) < MAX_PRIO)) {
pr_debug("%s: yielding SPU %d\n", __FUNCTION__, spu->number);
spu_deactivate(ctx);
ctx->state = SPU_STATE_SAVED;
need_yield = 1;
}
up_write(&ctx->state_sema);
if (unlikely(need_yield))
yield();
}
int __init spu_sched_init(void)
......
......@@ -51,6 +51,7 @@ struct spu_context {
struct kref kref;
wait_queue_head_t ibox_wq;
wait_queue_head_t wbox_wq;
wait_queue_head_t stop_wq;
struct fasync_struct *ibox_fasync;
struct fasync_struct *wbox_fasync;
struct spu_context_ops *ops;
......@@ -74,6 +75,8 @@ struct spu_context_ops {
void (*npc_write) (struct spu_context * ctx, u32 data);
u32(*status_read) (struct spu_context * ctx);
char*(*get_ls) (struct spu_context * ctx);
void (*runcntl_write) (struct spu_context * ctx, u32 data);
void (*runcntl_stop) (struct spu_context * ctx);
};
extern struct spu_context_ops spu_hw_ops;
......@@ -99,6 +102,7 @@ struct spu_context * alloc_spu_context(struct address_space *local_store);
void destroy_spu_context(struct kref *kref);
struct spu_context * get_spu_context(struct spu_context *ctx);
int put_spu_context(struct spu_context *ctx);
void spu_unmap_mappings(struct spu_context *ctx);
void spu_forget(struct spu_context *ctx);
void spu_acquire(struct spu_context *ctx);
......@@ -118,5 +122,6 @@ size_t spu_ibox_read(struct spu_context *ctx, u32 *data);
/* irq callback funcs. */
void spufs_ibox_callback(struct spu *spu);
void spufs_wbox_callback(struct spu *spu);
void spufs_stop_callback(struct spu *spu);
#endif
......@@ -135,9 +135,9 @@ struct spu {
spinlock_t register_lock;
u32 stop_code;
wait_queue_head_t stop_wq;
void (* wbox_callback)(struct spu *spu);
void (* ibox_callback)(struct spu *spu);
void (* stop_callback)(struct spu *spu);
char irq_c0[8];
char irq_c1[8];
......@@ -146,7 +146,8 @@ struct spu {
struct spu *spu_alloc(void);
void spu_free(struct spu *spu);
int spu_run(struct spu *spu);
int spu_irq_class_0_bottom(struct spu *spu);
int spu_irq_class_1_bottom(struct spu *spu);
extern struct spufs_calls {
asmlinkage long (*create_thread)(const char __user *name,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment