Real mode SLB miss
David Gibson
david at gibson.dropbear.id.au
Tue Aug 17 13:40:48 EST 2004
This probably wants testing on Power4 iSeries, at least, before
pushing to Andrew.
This patch makes the PPC64 SLB miss handler run in real mode (MMU off)
for its whole duration, on pSeries machines. Avoiding the rfid used
to turn relocation on saves some 70-80 cycles on Power4 and Power5.
Not having to save and restore SRR0 and SRR1 saves a few more, and
means we no longer need an extra save space (for r3) for the SLB miss.
Overall there's around a 27% speedup on a p630 (for a userspace SLB
miss).
Signed-off-by: David Gibson <david at gibson.dropbear.id.au>
Index: working-2.6/arch/ppc64/kernel/head.S
===================================================================
--- working-2.6.orig/arch/ppc64/kernel/head.S 2004-08-09 09:51:38.000000000 +1000
+++ working-2.6/arch/ppc64/kernel/head.S 2004-08-17 13:33:11.541468824 +1000
@@ -199,6 +199,7 @@
#define EX_R12 24
#define EX_R13 32
#define EX_SRR0 40
+#define EX_R3 40 /* SLB miss saves R3, but not SRR0 */
#define EX_DAR 48
#define EX_LR 48 /* SLB miss saves LR, but not DAR */
#define EX_DSISR 56
@@ -446,21 +447,13 @@
std r10,PACA_EXSLB+EX_R10(r13)
std r11,PACA_EXSLB+EX_R11(r13)
std r12,PACA_EXSLB+EX_R12(r13)
- std r3,PACASLBR3(r13)
+ std r3,PACA_EXSLB+EX_R3(r13)
mfspr r9,SPRG1
std r9,PACA_EXSLB+EX_R13(r13)
mfcr r9
- clrrdi r12,r13,32 /* get high part of &label */
- mfmsr r10
- mfspr r11,SRR0 /* save SRR0 */
- ori r12,r12,(.do_slb_miss)@l
- ori r10,r10,MSR_IR|MSR_DR /* DON'T set RI for SLB miss */
- mtspr SRR0,r12
mfspr r12,SRR1 /* and SRR1 */
- mtspr SRR1,r10
mfspr r3,DAR
- rfid
- b . /* prevent speculative execution */
+ b .do_slb_miss /* Rel. branch works in real mode */
STD_EXCEPTION_PSERIES(0x400, InstructionAccess)
@@ -474,21 +467,13 @@
std r10,PACA_EXSLB+EX_R10(r13)
std r11,PACA_EXSLB+EX_R11(r13)
std r12,PACA_EXSLB+EX_R12(r13)
- std r3,PACASLBR3(r13)
+ std r3,PACA_EXSLB+EX_R3(r13)
mfspr r9,SPRG1
std r9,PACA_EXSLB+EX_R13(r13)
mfcr r9
- clrrdi r12,r13,32 /* get high part of &label */
- mfmsr r10
- mfspr r11,SRR0 /* save SRR0 */
- ori r12,r12,(.do_slb_miss)@l
- ori r10,r10,MSR_IR|MSR_DR /* DON'T set RI for SLB miss */
- mtspr SRR0,r12
mfspr r12,SRR1 /* and SRR1 */
- mtspr SRR1,r10
- mr r3,r11 /* SRR0 is faulting address */
- rfid
- b . /* prevent speculative execution */
+ mfspr r3,SRR0 /* SRR0 is faulting address */
+ b .do_slb_miss /* Rel. branch works in real mode */
STD_EXCEPTION_PSERIES(0x500, HardwareInterrupt)
STD_EXCEPTION_PSERIES(0x600, Alignment)
@@ -630,8 +615,7 @@
DataAccessSLB_Iseries:
mtspr SPRG1,r13 /* save r13 */
EXCEPTION_PROLOG_ISERIES_1(PACA_EXSLB)
- std r3,PACASLBR3(r13)
- ld r11,PACALPPACA+LPPACASRR0(r13)
+ std r3,PACA_EXSLB+EX_R3(r13)
ld r12,PACALPPACA+LPPACASRR1(r13)
mfspr r3,DAR
b .do_slb_miss
@@ -642,10 +626,9 @@
InstructionAccessSLB_Iseries:
mtspr SPRG1,r13 /* save r13 */
EXCEPTION_PROLOG_ISERIES_1(PACA_EXSLB)
- std r3,PACASLBR3(r13)
- ld r11,PACALPPACA+LPPACASRR0(r13)
+ std r3,PACA_EXSLB+EX_R3(r13)
ld r12,PACALPPACA+LPPACASRR1(r13)
- mr r3,r11
+ ld r3,PACALPPACA+LPPACASRR0(r13)
b .do_slb_miss
MASKABLE_EXCEPTION_ISERIES(0x500, HardwareInterrupt)
@@ -1176,7 +1159,6 @@
mflr r10
stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
- std r11,PACA_EXSLB+EX_SRR0(r13) /* save SRR0 in exc. frame */
std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
bl .slb_allocate /* handle it */
@@ -1184,9 +1166,11 @@
/* All done -- return from exception. */
ld r10,PACA_EXSLB+EX_LR(r13)
- ld r3,PACASLBR3(r13)
+ ld r3,PACA_EXSLB+EX_R3(r13)
lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
- ld r11,PACA_EXSLB+EX_SRR0(r13) /* get saved SRR0 */
+#ifdef CONFIG_PPC_ISERIES
+ ld r11,PACALPPACA+LPPACASRR0(r13) /* get SRR0 value */
+#endif /* CONFIG_PPC_ISERIES */
mtlr r10
@@ -1199,8 +1183,10 @@
mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
.machine pop
+#ifdef CONFIG_PPC_ISERIES
mtspr SRR0,r11
mtspr SRR1,r12
+#endif /* CONFIG_PPC_ISERIES */
ld r9,PACA_EXSLB+EX_R9(r13)
ld r10,PACA_EXSLB+EX_R10(r13)
ld r11,PACA_EXSLB+EX_R11(r13)
Index: working-2.6/include/asm-ppc64/paca.h
===================================================================
--- working-2.6.orig/include/asm-ppc64/paca.h 2004-08-09 09:52:53.000000000 +1000
+++ working-2.6/include/asm-ppc64/paca.h 2004-08-17 13:33:35.437564584 +1000
@@ -78,7 +78,6 @@
u64 exmc[8]; /* used for machine checks */
u64 exslb[8]; /* used for SLB/segment table misses
* on the linear mapping */
- u64 slb_r3; /* spot to save R3 on SLB miss */
mm_context_t context;
u16 slb_cache[SLB_CACHE_ENTRIES];
u16 slb_cache_ptr;
Index: working-2.6/arch/ppc64/kernel/asm-offsets.c
===================================================================
--- working-2.6.orig/arch/ppc64/kernel/asm-offsets.c 2004-08-09 09:51:38.000000000 +1000
+++ working-2.6/arch/ppc64/kernel/asm-offsets.c 2004-08-17 13:33:59.597486608 +1000
@@ -93,7 +93,6 @@
DEFINE(PACASLBCACHE, offsetof(struct paca_struct, slb_cache));
DEFINE(PACASLBCACHEPTR, offsetof(struct paca_struct, slb_cache_ptr));
DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));
- DEFINE(PACASLBR3, offsetof(struct paca_struct, slb_r3));
#ifdef CONFIG_HUGETLB_PAGE
DEFINE(PACAHTLBSEGS, offsetof(struct paca_struct, context.htlb_segs));
#endif /* CONFIG_HUGETLB_PAGE */
--
David Gibson | For every complex problem there is a
david AT gibson.dropbear.id.au | solution which is simple, neat and
| wrong.
http://www.ozlabs.org/people/dgibson
** Sent via the linuxppc64-dev mail list. See http://lists.linuxppc.org/
More information about the Linuxppc64-dev
mailing list