[PATCH 09/10] powerpc/booke64: Critical and machine check exception support
Scott Wood
scottwood at freescale.com
Fri Mar 14 11:00:48 EST 2014
Add special state saving for critical and machine check exceptions.
Most of this code could be used to handle debug exceptions taken from
kernel space, but actually doing so is outside the scope of this patch.
The various critical and machine check exceptions now point to their
real handlers, rather than hanging the kernel.
Signed-off-by: Scott Wood <scottwood at freescale.com>
---
arch/powerpc/include/asm/paca.h | 7 +-
arch/powerpc/kernel/exceptions-64e.S | 343 +++++++++++++++++++++++++++++------
arch/powerpc/mm/tlb_nohash.c | 11 ++
3 files changed, 300 insertions(+), 61 deletions(-)
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 948f01a..8e956a0 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -116,8 +116,11 @@ struct paca_struct {
/* Shared by all threads of a core -- points to tcd of first thread */
struct tlb_core_data *tcd_ptr;
- /* We can have up to 3 levels of reentrancy in the TLB miss handler */
- u64 extlb[3][EX_TLB_SIZE / sizeof(u64)];
+ /*
+ * We can have up to 3 levels of reentrancy in the TLB miss handler,
+ * in each of four exception levels (normal, crit, mcheck, debug).
+ */
+ u64 extlb[12][EX_TLB_SIZE / sizeof(u64)];
u64 exmc[8]; /* used for machine checks */
u64 excrit[8]; /* used for crit interrupts */
u64 exdbg[8]; /* used for debug interrupts */
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index 2beb5bd..c1bee3c 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -34,20 +34,250 @@
* special interrupts from within a non-standard level will probably
* blow you up
*/
-#define SPECIAL_EXC_FRAME_SIZE INT_FRAME_SIZE
+#define SPECIAL_EXC_SRR0 0
+#define SPECIAL_EXC_SRR1 1
+#define SPECIAL_EXC_SPRG_GEN 2
+#define SPECIAL_EXC_SPRG_TLB 3
+#define SPECIAL_EXC_MAS0 4
+#define SPECIAL_EXC_MAS1 5
+#define SPECIAL_EXC_MAS2 6
+#define SPECIAL_EXC_MAS3 7
+#define SPECIAL_EXC_MAS6 8
+#define SPECIAL_EXC_MAS7 9
+#define SPECIAL_EXC_MAS5 10 /* E.HV only */
+#define SPECIAL_EXC_MAS8 11 /* E.HV only */
+#define SPECIAL_EXC_IRQHAPPENED 12
+#define SPECIAL_EXC_DEAR 13
+#define SPECIAL_EXC_ESR 14
+#define SPECIAL_EXC_SOFTE 15
+#define SPECIAL_EXC_CSRR0 16
+#define SPECIAL_EXC_CSRR1 17
+/* must be even to keep 16-byte stack alignment */
+#define SPECIAL_EXC_END 18
+
+#define SPECIAL_EXC_FRAME_SIZE (INT_FRAME_SIZE + SPECIAL_EXC_END * 8)
+#define SPECIAL_EXC_FRAME_OFFS (INT_FRAME_SIZE - 288)
+
+#define SPECIAL_EXC_STORE(reg, name) \
+ std reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
+
+#define SPECIAL_EXC_LOAD(reg, name) \
+ ld reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
+
+special_reg_save:
+ lbz r9,PACAIRQHAPPENED(r13)
+ RECONCILE_IRQ_STATE(r3,r4)
-/* Now we only store something to exception thread info */
-#define EXC_LEVEL_EXCEPTION_PROLOG(type) \
- ld r14,PACAKSAVE(r13); \
- CURRENT_THREAD_INFO(r14, r14); \
- CURRENT_THREAD_INFO(r15, r1); \
- ld r10,TI_FLAGS(r14); \
- std r10,TI_FLAGS(r15); \
- ld r10,TI_PREEMPT(r14); \
- std r10,TI_PREEMPT(r15); \
- ld r10,TI_TASK(r14); \
- std r10,TI_TASK(r15);
+ /*
+ * We only need (or have stack space) to save this stuff if
+ * we interrupted the kernel.
+ */
+ ld r3,_MSR(r1)
+ andi. r3,r3,MSR_PR
+ bnelr
+
+ /* Copy info into temporary exception thread info */
+ ld r11,PACAKSAVE(r13)
+ CURRENT_THREAD_INFO(r11, r11)
+ CURRENT_THREAD_INFO(r12, r1)
+ ld r10,TI_FLAGS(r11)
+ std r10,TI_FLAGS(r12)
+ ld r10,TI_PREEMPT(r11)
+ std r10,TI_PREEMPT(r12)
+ ld r10,TI_TASK(r11)
+ std r10,TI_TASK(r12)
+
+ /*
+ * Advance to the next TLB exception frame for handler
+ * types that don't do it automatically.
+ */
+ LOAD_REG_ADDR(r11,extlb_level_exc)
+ lwz r12,0(r11)
+ mfspr r10,SPRN_SPRG_TLB_EXFRAME
+ add r10,r10,r12
+ mtspr SPRN_SPRG_TLB_EXFRAME,r10
+
+ /*
+ * Save registers needed to allow nesting of certain exceptions
+ * (such as TLB misses) inside special exception levels
+ */
+ mfspr r10,SPRN_SRR0
+ SPECIAL_EXC_STORE(r10,SRR0)
+ mfspr r10,SPRN_SRR1
+ SPECIAL_EXC_STORE(r10,SRR1)
+ mfspr r10,SPRN_SPRG_GEN_SCRATCH
+ SPECIAL_EXC_STORE(r10,SPRG_GEN)
+ mfspr r10,SPRN_SPRG_TLB_SCRATCH
+ SPECIAL_EXC_STORE(r10,SPRG_TLB)
+ mfspr r10,SPRN_MAS0
+ SPECIAL_EXC_STORE(r10,MAS0)
+ mfspr r10,SPRN_MAS1
+ SPECIAL_EXC_STORE(r10,MAS1)
+ mfspr r10,SPRN_MAS2
+ SPECIAL_EXC_STORE(r10,MAS2)
+ mfspr r10,SPRN_MAS3
+ SPECIAL_EXC_STORE(r10,MAS3)
+ mfspr r10,SPRN_MAS6
+ SPECIAL_EXC_STORE(r10,MAS6)
+ mfspr r10,SPRN_MAS7
+ SPECIAL_EXC_STORE(r10,MAS7)
+BEGIN_FTR_SECTION
+ mfspr r10,SPRN_MAS5
+ SPECIAL_EXC_STORE(r10,MAS5)
+ mfspr r10,SPRN_MAS8
+ SPECIAL_EXC_STORE(r10,MAS8)
+
+ /* MAS5/8 could have inappropriate values if we interrupted KVM code */
+ li r10,0
+ mtspr SPRN_MAS5,r10
+ mtspr SPRN_MAS8,r10
+END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
+ SPECIAL_EXC_STORE(r9,IRQHAPPENED)
+
+ mfspr r10,SPRN_DEAR
+ SPECIAL_EXC_STORE(r10,DEAR)
+ mfspr r10,SPRN_ESR
+ SPECIAL_EXC_STORE(r10,ESR)
+
+ lbz r10,PACASOFTIRQEN(r13)
+ SPECIAL_EXC_STORE(r10,SOFTE)
+ ld r10,_NIP(r1)
+ SPECIAL_EXC_STORE(r10,CSRR0)
+ ld r10,_MSR(r1)
+ SPECIAL_EXC_STORE(r10,CSRR1)
+
+ blr
+
+ret_from_level_except:
+ ld r3,_MSR(r1)
+ andi. r3,r3,MSR_PR
+ beq 1f
+ b ret_from_except
+1:
+
+ LOAD_REG_ADDR(r11,extlb_level_exc)
+ lwz r12,0(r11)
+ mfspr r10,SPRN_SPRG_TLB_EXFRAME
+ sub r10,r10,r12
+ mtspr SPRN_SPRG_TLB_EXFRAME,r10
+
+ /*
+ * It's possible that the special level exception interrupted a
+ * TLB miss handler, and inserted the same entry that the
+ * interrupted handler was about to insert. On CPUs without TLB
+ * write conditional, this can result in a duplicate TLB entry.
+ * Wipe all non-bolted entries to be safe.
+ *
+ * Note that this doesn't protect against any TLB misses
+ * we may take accessing the stack from here to the end of
+ * the special level exception. It's not clear how we can
+ * reasonably protect against that, but only CPUs with
+ * neither TLB write conditional nor bolted kernel memory
+ * are affected. Do any such CPUs even exist?
+ */
+ PPC_TLBILX_ALL(0,R0)
+
+ REST_NVGPRS(r1)
+
+ SPECIAL_EXC_LOAD(r10,SRR0)
+ mtspr SPRN_SRR0,r10
+ SPECIAL_EXC_LOAD(r10,SRR1)
+ mtspr SPRN_SRR1,r10
+ SPECIAL_EXC_LOAD(r10,SPRG_GEN)
+ mtspr SPRN_SPRG_GEN_SCRATCH,r10
+ SPECIAL_EXC_LOAD(r10,SPRG_TLB)
+ mtspr SPRN_SPRG_TLB_SCRATCH,r10
+ SPECIAL_EXC_LOAD(r10,MAS0)
+ mtspr SPRN_MAS0,r10
+ SPECIAL_EXC_LOAD(r10,MAS1)
+ mtspr SPRN_MAS1,r10
+ SPECIAL_EXC_LOAD(r10,MAS2)
+ mtspr SPRN_MAS2,r10
+ SPECIAL_EXC_LOAD(r10,MAS3)
+ mtspr SPRN_MAS3,r10
+ SPECIAL_EXC_LOAD(r10,MAS6)
+ mtspr SPRN_MAS6,r10
+ SPECIAL_EXC_LOAD(r10,MAS7)
+ mtspr SPRN_MAS7,r10
+BEGIN_FTR_SECTION
+ SPECIAL_EXC_LOAD(r10,MAS5)
+ mtspr SPRN_MAS5,r10
+ SPECIAL_EXC_LOAD(r10,MAS8)
+ mtspr SPRN_MAS8,r10
+END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
+
+ lbz r6,PACASOFTIRQEN(r13)
+ ld r5,SOFTE(r1)
+
+ /* Interrupts had better not already be enabled... */
+ twnei r6,0
+
+ cmpwi cr0,r5,0
+ beq 1f
+ TRACE_ENABLE_INTS
+ stb r5,PACASOFTIRQEN(r13)
+1:
+ /*
+ * Restore PACAIRQHAPPENED rather than setting it based on
+ * the return MSR[EE], since we could have interrupted
+ * __check_irq_replay() or other inconsistent transitory
+ * states that must remain that way.
+ */
+ SPECIAL_EXC_LOAD(r10,IRQHAPPENED)
+ stb r10,PACAIRQHAPPENED(r13)
+
+ SPECIAL_EXC_LOAD(r10,DEAR)
+ mtspr SPRN_DEAR,r10
+ SPECIAL_EXC_LOAD(r10,ESR)
+ mtspr SPRN_ESR,r10
+
+ stdcx. r0,0,r1 /* to clear the reservation */
+
+ REST_4GPRS(2, r1)
+ REST_4GPRS(6, r1)
+
+ ld r10,_CTR(r1)
+ ld r11,_XER(r1)
+ mtctr r10
+ mtxer r11
+
+ blr
+
+.macro ret_from_level srr0 srr1 paca_ex scratch
+ bl ret_from_level_except
+
+ ld r10,_LINK(r1)
+ ld r11,_CCR(r1)
+ ld r0,GPR13(r1)
+ mtlr r10
+ mtcr r11
+
+ ld r10,GPR10(r1)
+ ld r11,GPR11(r1)
+ ld r12,GPR12(r1)
+ mtspr \scratch,r0
+
+ std r10,\paca_ex+EX_R10(r13);
+ std r11,\paca_ex+EX_R11(r13);
+ ld r10,_NIP(r1)
+ ld r11,_MSR(r1)
+ ld r0,GPR0(r1)
+ ld r1,GPR1(r1)
+ mtspr \srr0,r10
+ mtspr \srr1,r11
+ ld r10,\paca_ex+EX_R10(r13)
+ ld r11,\paca_ex+EX_R11(r13)
+ mfspr r13,\scratch
+.endm
+
+ret_from_crit_except:
+ ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH
+ rfci
+
+ret_from_mc_except:
+ ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH
+ rfmci
/* Exception prolog code for all exceptions */
#define EXCEPTION_PROLOG(n, intnum, type, addition) \
@@ -81,22 +311,19 @@
#define CRIT_SET_KSTACK \
ld r1,PACA_CRIT_STACK(r13); \
- subi r1,r1,SPECIAL_EXC_FRAME_SIZE; \
- EXC_LEVEL_EXCEPTION_PROLOG(CRIT);
+ subi r1,r1,SPECIAL_EXC_FRAME_SIZE
#define SPRN_CRIT_SRR0 SPRN_CSRR0
#define SPRN_CRIT_SRR1 SPRN_CSRR1
#define DBG_SET_KSTACK \
ld r1,PACA_DBG_STACK(r13); \
- subi r1,r1,SPECIAL_EXC_FRAME_SIZE; \
- EXC_LEVEL_EXCEPTION_PROLOG(DBG);
+ subi r1,r1,SPECIAL_EXC_FRAME_SIZE
#define SPRN_DBG_SRR0 SPRN_DSRR0
#define SPRN_DBG_SRR1 SPRN_DSRR1
#define MC_SET_KSTACK \
ld r1,PACA_MC_STACK(r13); \
- subi r1,r1,SPECIAL_EXC_FRAME_SIZE; \
- EXC_LEVEL_EXCEPTION_PROLOG(MC);
+ subi r1,r1,SPECIAL_EXC_FRAME_SIZE
#define SPRN_MC_SRR0 SPRN_MCSRR0
#define SPRN_MC_SRR1 SPRN_MCSRR1
@@ -322,27 +549,25 @@ interrupt_end_book3e:
START_EXCEPTION(critical_input);
CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON_CRIT(0x100)
-// INTS_DISABLE
-// bl special_reg_save_crit
-// CHECK_NAPPING();
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// bl .critical_exception
-// b ret_from_crit_except
- b .
+ EXCEPTION_COMMON_CRIT(0x100)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl .unknown_exception
+ b ret_from_crit_except
/* Machine Check Interrupt */
START_EXCEPTION(machine_check);
MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON_MC(0x000)
-// INTS_DISABLE
-// bl special_reg_save_mc
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// CHECK_NAPPING();
-// bl .machine_check_exception
-// b ret_from_mc_except
- b .
+ EXCEPTION_COMMON_MC(0x000)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl .machine_check_exception
+ b ret_from_mc_except
/* Data Storage Interrupt */
START_EXCEPTION(data_storage)
@@ -461,14 +686,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
START_EXCEPTION(watchdog);
CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON_CRIT(0x9f0)
-// INTS_DISABLE
-// bl special_reg_save_crit
-// CHECK_NAPPING();
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// bl .unknown_exception
-// b ret_from_crit_except
- b .
+ EXCEPTION_COMMON_CRIT(0x9f0)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+#ifdef CONFIG_BOOKE_WDT
+ bl .WatchdogException
+#else
+ bl .unknown_exception
+#endif
+ b ret_from_crit_except
/* System Call Interrupt */
START_EXCEPTION(system_call)
@@ -541,7 +769,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
*/
mfspr r14,SPRN_DBSR
EXCEPTION_COMMON_CRIT(0xd00)
- INTS_DISABLE
std r14,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
mr r4,r14
@@ -634,14 +861,13 @@ kernel_dbg_exc:
START_EXCEPTION(doorbell_crit);
CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON_CRIT(0x2a0)
-// INTS_DISABLE
-// bl special_reg_save_crit
-// CHECK_NAPPING();
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// bl .doorbell_critical_exception
-// b ret_from_crit_except
- b .
+ EXCEPTION_COMMON_CRIT(0x2a0)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl .unknown_exception
+ b ret_from_crit_except
/*
* Guest doorbell interrupt
@@ -661,14 +887,13 @@ kernel_dbg_exc:
START_EXCEPTION(guest_doorbell_crit);
CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON_CRIT(0x2e0)
-// INTS_DISABLE
-// bl special_reg_save_crit
-// CHECK_NAPPING();
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// bl .guest_doorbell_critical_exception
-// b ret_from_crit_except
- b .
+ EXCEPTION_COMMON_CRIT(0x2e0)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl .unknown_exception
+ b ret_from_crit_except
/* Hypervisor call */
START_EXCEPTION(hypercall);
diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
index b37a58e..ae3d5b7 100644
--- a/arch/powerpc/mm/tlb_nohash.c
+++ b/arch/powerpc/mm/tlb_nohash.c
@@ -144,6 +144,15 @@ int mmu_vmemmap_psize; /* Page size used for the virtual mem map */
int book3e_htw_mode; /* HW tablewalk? Value is PPC_HTW_* */
unsigned long linear_map_top; /* Top of linear mapping */
+
+/*
+ * Number of bytes to add to SPRN_SPRG_TLB_EXFRAME on crit/mcheck/debug
+ * exceptions. This is used for bolted and e6500 TLB miss handlers which
+ * do not modify this SPRG in the TLB miss code; for other TLB miss handlers,
+ * this is set to zero.
+ */
+int extlb_level_exc;
+
#endif /* CONFIG_PPC64 */
#ifdef CONFIG_PPC_FSL_BOOK3E
@@ -559,6 +568,7 @@ static void setup_mmu_htw(void)
break;
#ifdef CONFIG_PPC_FSL_BOOK3E
case PPC_HTW_E6500:
+ extlb_level_exc = EX_TLB_SIZE;
patch_exception(0x1c0, exc_data_tlb_miss_e6500_book3e);
patch_exception(0x1e0, exc_instruction_tlb_miss_e6500_book3e);
break;
@@ -652,6 +662,7 @@ static void __early_init_mmu(int boot_cpu)
memblock_enforce_memory_limit(linear_map_top);
if (book3e_htw_mode == PPC_HTW_NONE) {
+ extlb_level_exc = EX_TLB_SIZE;
patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
patch_exception(0x1e0,
exc_instruction_tlb_miss_bolted_book3e);
--
1.8.3.2
More information about the Linuxppc-dev
mailing list