[Cbe-oss-dev] [PATCH 13/13] spufs: rework class 0 and 1 interrupt handling
Jeremy Kerr
jk at ozlabs.org
Mon Nov 19 14:36:53 EST 2007
Based on an original patch from Arnd Bergmann <arnd.bergman at de.ibm.com>
Currently, spu contexts need to be loaded to the SPU in order to take
class 0 and class 1 exceptions.
This change makes the actual interrupt-handlers much simpler (ie,
set the exception information in the context save area), and defers the
handling code to the spufs_handle_class[01] functions, called from
spufs_run_spu.
This should improve the concurrency of the spu scheduling leading to
greater SPU utilization when SPUs are overcommited.
Signed-off-by: Jeremy Kerr <jk at ozlabs.org>
---
arch/powerpc/platforms/cell/spu_base.c | 57 +--------------
arch/powerpc/platforms/cell/spufs/fault.c | 107 +++++++++++++++++------------
arch/powerpc/platforms/cell/spufs/run.c | 50 +++++++++----
arch/powerpc/platforms/cell/spufs/sched.c | 2
arch/powerpc/platforms/cell/spufs/spufs.h | 1
arch/powerpc/platforms/cell/spufs/switch.c | 7 -
include/asm-powerpc/spu.h | 3
include/asm-powerpc/spu_csa.h | 2
8 files changed, 108 insertions(+), 121 deletions(-)
diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c
index b3bcdf6..036aef9 100644
--- a/arch/powerpc/platforms/cell/spu_base.c
+++ b/arch/powerpc/platforms/cell/spu_base.c
@@ -119,27 +119,6 @@ void spu_associate_mm(struct spu *spu, struct mm_struct *mm)
}
EXPORT_SYMBOL_GPL(spu_associate_mm);
-static int __spu_trap_invalid_dma(struct spu *spu)
-{
- pr_debug("%s\n", __FUNCTION__);
- spu->dma_callback(spu, SPE_EVENT_INVALID_DMA);
- return 0;
-}
-
-static int __spu_trap_dma_align(struct spu *spu)
-{
- pr_debug("%s\n", __FUNCTION__);
- spu->dma_callback(spu, SPE_EVENT_DMA_ALIGNMENT);
- return 0;
-}
-
-static int __spu_trap_error(struct spu *spu)
-{
- pr_debug("%s\n", __FUNCTION__);
- spu->dma_callback(spu, SPE_EVENT_SPE_ERROR);
- return 0;
-}
-
static void spu_restart_dma(struct spu *spu)
{
struct spu_priv2 __iomem *priv2 = spu->priv2;
@@ -239,10 +218,12 @@ static int __spu_trap_data_map(struct spu *spu, unsigned long ea, u64 dsisr)
return 1;
}
+ spu->class_0_pending = 0;
spu->dar = ea;
spu->dsisr = dsisr;
- mb();
+
spu->stop_callback(spu);
+
return 0;
}
@@ -322,12 +303,13 @@ spu_irq_class_0(int irq, void *data)
spu = data;
+ spin_lock(&spu->register_lock);
mask = spu_int_mask_get(spu, 0);
- stat = spu_int_stat_get(spu, 0);
- stat &= mask;
+ stat = spu_int_stat_get(spu, 0) & mask;
- spin_lock(&spu->register_lock);
spu->class_0_pending |= stat;
+ spu->dsisr = spu_mfc_dsisr_get(spu);
+ spu->dar = spu_mfc_dar_get(spu);
spin_unlock(&spu->register_lock);
spu->stop_callback(spu);
@@ -337,31 +319,6 @@ spu_irq_class_0(int irq, void *data)
return IRQ_HANDLED;
}
-int
-spu_irq_class_0_bottom(struct spu *spu)
-{
- unsigned long flags;
- unsigned long stat;
-
- spin_lock_irqsave(&spu->register_lock, flags);
- stat = spu->class_0_pending;
- spu->class_0_pending = 0;
-
- if (stat & CLASS0_DMA_ALIGNMENT_INTR)
- __spu_trap_dma_align(spu);
-
- if (stat & CLASS0_INVALID_DMA_COMMAND_INTR)
- __spu_trap_invalid_dma(spu);
-
- if (stat & CLASS0_SPU_ERROR_INTR)
- __spu_trap_error(spu);
-
- spin_unlock_irqrestore(&spu->register_lock, flags);
-
- return (stat & CLASS0_INTR_MASK) ? -EIO : 0;
-}
-EXPORT_SYMBOL_GPL(spu_irq_class_0_bottom);
-
static irqreturn_t
spu_irq_class_1(int irq, void *data)
{
diff --git a/arch/powerpc/platforms/cell/spufs/fault.c b/arch/powerpc/platforms/cell/spufs/fault.c
index 0635f29..720e111 100644
--- a/arch/powerpc/platforms/cell/spufs/fault.c
+++ b/arch/powerpc/platforms/cell/spufs/fault.c
@@ -28,46 +28,69 @@
#include "spufs.h"
-static void spufs_handle_dma_error(struct spu_context *ctx,
+/**
+ * Handle an SPE event, depending on context SPU_CREATE_EVENTS_ENABLED flag.
+ *
+ * If the context was created with events, we just set the return event.
+ * Otherwise, send an appropriate signal to the process.
+ */
+static void spufs_handle_event(struct spu_context *ctx,
unsigned long ea, int type)
{
+ siginfo_t info;
+
if (ctx->flags & SPU_CREATE_EVENTS_ENABLED) {
ctx->event_return |= type;
wake_up_all(&ctx->stop_wq);
- } else {
- siginfo_t info;
- memset(&info, 0, sizeof(info));
-
- switch (type) {
- case SPE_EVENT_INVALID_DMA:
- info.si_signo = SIGBUS;
- info.si_code = BUS_OBJERR;
- break;
- case SPE_EVENT_SPE_DATA_STORAGE:
- info.si_signo = SIGBUS;
- info.si_addr = (void __user *)ea;
- info.si_code = BUS_ADRERR;
- break;
- case SPE_EVENT_DMA_ALIGNMENT:
- info.si_signo = SIGBUS;
- /* DAR isn't set for an alignment fault :( */
- info.si_code = BUS_ADRALN;
- break;
- case SPE_EVENT_SPE_ERROR:
- info.si_signo = SIGILL;
- info.si_addr = (void __user *)(unsigned long)
- ctx->ops->npc_read(ctx) - 4;
- info.si_code = ILL_ILLOPC;
- break;
- }
- if (info.si_signo)
- force_sig_info(info.si_signo, &info, current);
+ return;
}
+
+ memset(&info, 0, sizeof(info));
+
+ switch (type) {
+ case SPE_EVENT_INVALID_DMA:
+ info.si_signo = SIGBUS;
+ info.si_code = BUS_OBJERR;
+ break;
+ case SPE_EVENT_SPE_DATA_STORAGE:
+ info.si_signo = SIGBUS;
+ info.si_addr = (void __user *)ea;
+ info.si_code = BUS_ADRERR;
+ break;
+ case SPE_EVENT_DMA_ALIGNMENT:
+ info.si_signo = SIGBUS;
+ /* DAR isn't set for an alignment fault :( */
+ info.si_code = BUS_ADRALN;
+ break;
+ case SPE_EVENT_SPE_ERROR:
+ info.si_signo = SIGILL;
+ info.si_addr = (void __user *)(unsigned long)
+ ctx->ops->npc_read(ctx) - 4;
+ info.si_code = ILL_ILLOPC;
+ break;
+ }
+
+ if (info.si_signo)
+ force_sig_info(info.si_signo, &info, current);
}
-void spufs_dma_callback(struct spu *spu, int type)
+int spufs_handle_class0(struct spu_context *ctx)
{
- spufs_handle_dma_error(spu->ctx, spu->dar, type);
+ unsigned long stat = ctx->csa.class_0_pending & CLASS0_INTR_MASK;
+
+ if (likely(!stat))
+ return 0;
+
+ if (stat & CLASS0_DMA_ALIGNMENT_INTR)
+ spufs_handle_event(ctx, ctx->csa.dar, SPE_EVENT_DMA_ALIGNMENT);
+
+ if (stat & CLASS0_INVALID_DMA_COMMAND_INTR)
+ spufs_handle_event(ctx, ctx->csa.dar, SPE_EVENT_INVALID_DMA);
+
+ if (stat & CLASS0_SPU_ERROR_INTR)
+ spufs_handle_event(ctx, ctx->csa.dar, SPE_EVENT_SPE_ERROR);
+
+ return -EIO;
}
/*
@@ -95,16 +118,8 @@ int spufs_handle_class1(struct spu_context *ctx)
* in time, we can still expect to get the same fault
* the immediately after the context restore.
*/
- if (ctx->state == SPU_STATE_RUNNABLE) {
- ea = ctx->spu->dar;
- dsisr = ctx->spu->dsisr;
- ctx->spu->dar= ctx->spu->dsisr = 0;
- } else {
- ea = ctx->csa.priv1.mfc_dar_RW;
- dsisr = ctx->csa.priv1.mfc_dsisr_RW;
- ctx->csa.priv1.mfc_dar_RW = 0;
- ctx->csa.priv1.mfc_dsisr_RW = 0;
- }
+ ea = ctx->csa.dar;
+ dsisr = ctx->csa.dsisr;
if (!(dsisr & (MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED)))
return 0;
@@ -132,6 +147,14 @@ int spufs_handle_class1(struct spu_context *ctx)
ret = spu_handle_mm_fault(current->mm, ea, dsisr, &flt);
spu_acquire(ctx);
+
+ /*
+ * Clear dsisr under ctxt lock after handling the fault, so that
+ * time slicing will not preempt the context while the page fault
+ * handler is running. Context switch code removes mappings.
+ */
+ ctx->csa.dar = ctx->csa.dsisr = 0;
+
/*
* If we handled the fault successfully and are in runnable
* state, restart the DMA.
@@ -152,7 +175,7 @@ int spufs_handle_class1(struct spu_context *ctx)
if (ctx->spu)
ctx->ops->restart_dma(ctx);
} else
- spufs_handle_dma_error(ctx, ea, SPE_EVENT_SPE_DATA_STORAGE);
+ spufs_handle_event(ctx, ea, SPE_EVENT_SPE_DATA_STORAGE);
spuctx_switch_state(ctx, SPU_UTIL_SYSTEM);
return ret;
diff --git a/arch/powerpc/platforms/cell/spufs/run.c b/arch/powerpc/platforms/cell/spufs/run.c
index b3cc1dd..3b3de6c 100644
--- a/arch/powerpc/platforms/cell/spufs/run.c
+++ b/arch/powerpc/platforms/cell/spufs/run.c
@@ -15,7 +15,30 @@ void spufs_stop_callback(struct spu *spu)
{
struct spu_context *ctx = spu->ctx;
- wake_up_all(&ctx->stop_wq);
+ /*
+ * It should be impossible to preempt a context while an exception
+ * is being processed, since the context switch code is specially
+ * coded to deal with interrupts ... But, just in case, sanity check
+ * the context pointer. It is OK to return doing nothing since
+ * the exception will be regenerated when the context is resumed.
+ */
+ if (ctx) {
+ /* Copy exception arguments into module specific structure */
+ ctx->csa.class_0_pending = spu->class_0_pending;
+ ctx->csa.dsisr = spu->dsisr;
+ ctx->csa.dar = spu->dar;
+
+ /* ensure that the exception status has hit memory before a
+ * thread waiting on the context's stop queue is woken */
+ smp_wmb();
+
+ wake_up_all(&ctx->stop_wq);
+ }
+
+ /* Clear callback arguments from spu structure */
+ spu->class_0_pending = 0;
+ spu->dsisr = 0;
+ spu->dar = 0;
}
static inline int spu_stopped(struct spu_context *ctx, u32 *stat)
@@ -29,9 +52,9 @@ static inline int spu_stopped(struct spu_context *ctx, u32 *stat)
if (ctx->state != SPU_STATE_RUNNABLE ||
test_bit(SPU_SCHED_NOTIFY_ACTIVE, &ctx->sched_flags))
return 1;
- pte_fault = spu->dsisr &
+ pte_fault = ctx->csa.dsisr &
(MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED);
- return (!(*stat & SPU_STATUS_RUNNING) || pte_fault || spu->class_0_pending) ?
+ return (!(*stat & SPU_STATUS_RUNNING) || pte_fault || ctx->csa.class_0_pending) ?
1 : 0;
}
@@ -287,18 +310,6 @@ static int spu_process_callback(struct spu_context *ctx)
return ret;
}
-static inline int spu_process_events(struct spu_context *ctx)
-{
- struct spu *spu = ctx->spu;
- int ret = 0;
-
- if (spu->class_0_pending)
- ret = spu_irq_class_0_bottom(spu);
- if (!ret && signal_pending(current))
- ret = -ERESTARTSYS;
- return ret;
-}
-
long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *event)
{
int ret;
@@ -364,13 +375,20 @@ long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *event)
if (ret)
break;
+ ret = spufs_handle_class0(ctx);
+ if (ret)
+ break;
+
if (unlikely(ctx->state != SPU_STATE_RUNNABLE)) {
ret = spu_reacquire_runnable(ctx, npc, &status);
if (ret)
goto out2;
continue;
}
- ret = spu_process_events(ctx);
+
+ if (signal_pending(current))
+ ret = -ERESTARTSYS;
+
} while (!ret && !(status & (SPU_STATUS_STOPPED_BY_STOP |
SPU_STATUS_STOPPED_BY_HALT |
diff --git a/arch/powerpc/platforms/cell/spufs/sched.c b/arch/powerpc/platforms/cell/spufs/sched.c
index 52215aa..82ea576 100644
--- a/arch/powerpc/platforms/cell/spufs/sched.c
+++ b/arch/powerpc/platforms/cell/spufs/sched.c
@@ -245,7 +245,6 @@ static void spu_bind_context(struct spu *spu, struct spu_context *ctx)
spu->wbox_callback = spufs_wbox_callback;
spu->stop_callback = spufs_stop_callback;
spu->mfc_callback = spufs_mfc_callback;
- spu->dma_callback = spufs_dma_callback;
mb();
spu_unmap_mappings(ctx);
spu_restore(&ctx->csa, spu);
@@ -433,7 +432,6 @@ static void spu_unbind_context(struct spu *spu, struct spu_context *ctx)
spu->wbox_callback = NULL;
spu->stop_callback = NULL;
spu->mfc_callback = NULL;
- spu->dma_callback = NULL;
spu_associate_mm(spu, NULL);
spu->pid = 0;
spu->tgid = 0;
diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h
index afdddbc..eaab1b2 100644
--- a/arch/powerpc/platforms/cell/spufs/spufs.h
+++ b/arch/powerpc/platforms/cell/spufs/spufs.h
@@ -222,6 +222,7 @@ void spu_gang_add_ctx(struct spu_gang *gang, struct spu_context *ctx);
/* fault handling */
int spufs_handle_class1(struct spu_context *ctx);
+int spufs_handle_class0(struct spu_context *ctx);
/* affinity */
struct spu *affinity_check(struct spu_context *ctx);
diff --git a/arch/powerpc/platforms/cell/spufs/switch.c b/arch/powerpc/platforms/cell/spufs/switch.c
index c9600e8..1a7d7a0 100644
--- a/arch/powerpc/platforms/cell/spufs/switch.c
+++ b/arch/powerpc/platforms/cell/spufs/switch.c
@@ -2077,10 +2077,6 @@ int spu_save(struct spu_state *prev, struct spu *spu)
int rc;
acquire_spu_lock(spu); /* Step 1. */
- prev->dar = spu->dar;
- prev->dsisr = spu->dsisr;
- spu->dar = 0;
- spu->dsisr = 0;
rc = __do_spu_save(prev, spu); /* Steps 2-53. */
release_spu_lock(spu);
if (rc != 0 && rc != 2 && rc != 6) {
@@ -2107,9 +2103,6 @@ int spu_restore(struct spu_state *new, struct spu *spu)
acquire_spu_lock(spu);
harvest(NULL, spu);
spu->slb_replace = 0;
- new->dar = 0;
- new->dsisr = 0;
- spu->class_0_pending = 0;
rc = __do_spu_restore(new, spu);
release_spu_lock(spu);
if (rc) {
diff --git a/include/asm-powerpc/spu.h b/include/asm-powerpc/spu.h
index 770d8da..fc16353 100644
--- a/include/asm-powerpc/spu.h
+++ b/include/asm-powerpc/spu.h
@@ -146,7 +146,6 @@ struct spu {
void (* ibox_callback)(struct spu *spu);
void (* stop_callback)(struct spu *spu);
void (* mfc_callback)(struct spu *spu);
- void (* dma_callback)(struct spu *spu, int type);
char irq_c0[8];
char irq_c1[8];
@@ -197,8 +196,6 @@ struct cbe_spu_info {
extern struct cbe_spu_info cbe_spu_info[];
void spu_init_channels(struct spu *spu);
-int spu_irq_class_0_bottom(struct spu *spu);
-int spu_irq_class_1_bottom(struct spu *spu);
void spu_irq_setaffinity(struct spu *spu, int cpu);
void spu_setup_kernel_slbs(struct spu *spu, struct spu_lscsa *lscsa,
diff --git a/include/asm-powerpc/spu_csa.h b/include/asm-powerpc/spu_csa.h
index f54dc88..a8dbbec 100644
--- a/include/asm-powerpc/spu_csa.h
+++ b/include/asm-powerpc/spu_csa.h
@@ -254,7 +254,7 @@ struct spu_state {
u64 spu_chnldata_RW[32];
u32 spu_mailbox_data[4];
u32 pu_mailbox_data[1];
- u64 dar, dsisr;
+ u64 dar, dsisr, class_0_pending;
unsigned long suspend_time;
spinlock_t register_lock;
};
More information about the cbe-oss-dev
mailing list