[PATCH 4/7] powerpc/mm: 64-bit: Don't load PACA in normal TLB miss exceptions
Scott Wood
scottwood at freescale.com
Thu May 19 07:05:33 EST 2011
Load it only when needed, in recursive/linear/indirect faults,
and in the stats code.
Signed-off-by: Scott Wood <scottwood at freescale.com>
---
arch/powerpc/include/asm/exception-64e.h | 28 +++++++++---------
arch/powerpc/mm/tlb_low_64e.S | 43 +++++++++++++++++------------
2 files changed, 39 insertions(+), 32 deletions(-)
diff --git a/arch/powerpc/include/asm/exception-64e.h b/arch/powerpc/include/asm/exception-64e.h
index 6921261..9b57a27 100644
--- a/arch/powerpc/include/asm/exception-64e.h
+++ b/arch/powerpc/include/asm/exception-64e.h
@@ -80,9 +80,9 @@ exc_##label##_book3e:
*
* This prolog handles re-entrancy (up to 3 levels supported in the PACA
* though we currently don't test for overflow). It provides you with a
- * re-entrancy safe working space of r10...r16 and CR with r12 being used
- * as the exception area pointer in the PACA for that level of re-entrancy
- * and r13 containing the PACA pointer.
+ * re-entrancy safe working space of r10...r16 (except r13) and CR with r12
+ * being used as the exception area pointer in the PACA for that level of
+ * re-entrancy.
*
* SRR0 and SRR1 are saved, but DEAR and ESR are not, since they don't apply
* as-is for instruction exceptions. It's up to the actual exception code
@@ -95,8 +95,6 @@ exc_##label##_book3e:
mfcr r10; \
std r11,EX_TLB_R11(r12); \
mfspr r11,SPRN_SPRG_TLB_SCRATCH; \
- std r13,EX_TLB_R13(r12); \
- ld r13,EX_TLB_PACA(r12); \
std r14,EX_TLB_R14(r12); \
addi r14,r12,EX_TLB_SIZE; \
std r15,EX_TLB_R15(r12); \
@@ -135,7 +133,6 @@ exc_##label##_book3e:
mtspr SPRN_SPRG_TLB_EXFRAME,freg; \
ld r11,EX_TLB_R11(r12); \
mtcr r14; \
- ld r13,EX_TLB_R13(r12); \
ld r14,EX_TLB_R14(r12); \
mtspr SPRN_SRR0,r15; \
ld r15,EX_TLB_R15(r12); \
@@ -148,11 +145,13 @@ exc_##label##_book3e:
TLB_MISS_RESTORE(r12)
#define TLB_MISS_EPILOG_ERROR \
- addi r12,r13,PACA_EXTLB; \
+ ld r10,EX_TLB_PACA(r12); \
+ addi r12,r10,PACA_EXTLB; \
TLB_MISS_RESTORE(r12)
#define TLB_MISS_EPILOG_ERROR_SPECIAL \
- addi r11,r13,PACA_EXTLB; \
+ ld r10,EX_TLB_PACA(r12); \
+ addi r11,r10,PACA_EXTLB; \
TLB_MISS_RESTORE(r11)
#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
@@ -160,25 +159,26 @@ exc_##label##_book3e:
mflr r10; \
std r8,EX_TLB_R8(r12); \
std r9,EX_TLB_R9(r12); \
- std r10,EX_TLB_LR(r12);
+ std r10,EX_TLB_LR(r12); \
+ ld r9,EX_TLB_PACA(r12);
#define TLB_MISS_RESTORE_STATS \
ld r16,EX_TLB_LR(r12); \
ld r9,EX_TLB_R9(r12); \
ld r8,EX_TLB_R8(r12); \
mtlr r16;
#define TLB_MISS_STATS_D(name) \
- addi r9,r13,MMSTAT_DSTATS+name; \
+ addi r9,r9,MMSTAT_DSTATS+name; \
bl .tlb_stat_inc;
#define TLB_MISS_STATS_I(name) \
- addi r9,r13,MMSTAT_ISTATS+name; \
+ addi r9,r9,MMSTAT_ISTATS+name; \
bl .tlb_stat_inc;
#define TLB_MISS_STATS_X(name) \
- ld r8,PACA_EXTLB+EX_TLB_ESR(r13); \
+ ld r8,PACA_EXTLB+EX_TLB_ESR(r9); \
cmpdi cr2,r8,-1; \
beq cr2,61f; \
- addi r9,r13,MMSTAT_DSTATS+name; \
+ addi r9,r9,MMSTAT_DSTATS+name; \
b 62f; \
-61: addi r9,r13,MMSTAT_ISTATS+name; \
+61: addi r9,r9,MMSTAT_ISTATS+name; \
62: bl .tlb_stat_inc;
#define TLB_MISS_STATS_SAVE_INFO \
std r14,EX_TLB_ESR(r12); /* save ESR */ \
diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S
index 17726d3..922fece 100644
--- a/arch/powerpc/mm/tlb_low_64e.S
+++ b/arch/powerpc/mm/tlb_low_64e.S
@@ -160,7 +160,6 @@
* r16 = faulting address
* r15 = region ID
* r14 = crap (free to use)
- * r13 = PACA
* r12 = TLB exception frame in PACA
* r11 = PTE permission mask
* r10 = crap (free to use)
@@ -299,8 +298,7 @@ normal_tlb_miss_access_fault:
*
* r16 = virtual page table faulting address
* r15 = region (top 4 bits of address)
- * r14 = crap (free to use)
- * r13 = PACA
+ * r14 = crap (we load with PACA)
* r12 = TLB exception frame in PACA
* r11 = crap (free to use)
* r10 = crap (free to use)
@@ -318,6 +316,8 @@ normal_tlb_miss_access_fault:
* so we could probably optimize things a bit
*/
virt_page_table_tlb_miss:
+ ld r14,EX_TLB_PACA(r12)
+
/* Are we hitting a kernel page table ? */
andi. r10,r15,0x8
@@ -325,7 +325,7 @@ virt_page_table_tlb_miss:
* and we happen to have the swapper_pg_dir at offset 8 from the user
* pgdir in the PACA :-).
*/
- add r11,r10,r13
+ add r11,r10,r14
/* If kernel, we need to clear MAS1 TID */
beq 1f
@@ -417,12 +417,12 @@ virt_page_table_tlb_miss_done:
* offset the return address by -4 in order to replay the tlbsrx
* instruction there
*/
- subf r10,r13,r12
+ subf r10,r14,r12
cmpldi cr0,r10,PACA_EXTLB+EX_TLB_SIZE
bne- 1f
- ld r11,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13)
+ ld r11,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r14)
addi r10,r11,-4
- std r10,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13)
+ std r10,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r14)
1:
END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV)
/* Return to caller, normal case */
@@ -449,13 +449,13 @@ virt_page_table_tlb_miss_fault:
* level as well. Since we are doing that, we don't need to clear or
* restore the TLB reservation neither.
*/
- subf r10,r13,r12
+ subf r10,r14,r12
cmpldi cr0,r10,PACA_EXTLB+EX_TLB_SIZE
bne- virt_page_table_tlb_miss_whacko_fault
/* We dig the original DEAR and ESR from slot 0 */
- ld r15,EX_TLB_DEAR+PACA_EXTLB(r13)
- ld r16,EX_TLB_ESR+PACA_EXTLB(r13)
+ ld r15,EX_TLB_DEAR+PACA_EXTLB(r14)
+ ld r16,EX_TLB_ESR+PACA_EXTLB(r14)
/* We check for the "special" ESR value for instruction faults */
cmpdi cr0,r16,-1
@@ -489,6 +489,8 @@ virt_page_table_tlb_miss_whacko_fault:
START_EXCEPTION(data_tlb_miss_htw)
TLB_MISS_PROLOG
+ ld r15,EX_TLB_PACA(r12)
+
/* Now we handle the fault proper. We only save DEAR in normal
* fault case since that's the only interesting values here.
* We could probably also optimize by not saving SRR0/1 in the
@@ -503,8 +505,12 @@ virt_page_table_tlb_miss_whacko_fault:
/* We do the user/kernel test for the PID here along with the RW test
*/
+ /* The cool thing now is that r11 contains 0 for user and 8 for kernel,
+ * and we happen to have the swapper_pg_dir at offset 8 from the user
+ * pgdir in the PACA :-).
+ */
cmpldi cr0,r11,0 /* Check for user region */
- ld r15,PACAPGD(r13) /* Load user pgdir */
+ add r15,r15,r11
beq htw_tlb_miss
/* XXX replace the RMW cycles with immediate loads + writes */
@@ -512,7 +518,6 @@ virt_page_table_tlb_miss_whacko_fault:
cmpldi cr0,r11,8 /* Check for vmalloc region */
rlwinm r10,r10,0,16,1 /* Clear TID */
mtspr SPRN_MAS1,r10
- ld r15,PACA_KERNELPGD(r13) /* Load kernel pgdir */
beq+ htw_tlb_miss
/* We got a crappy address, just fault with whatever DEAR and ESR
@@ -526,6 +531,8 @@ virt_page_table_tlb_miss_whacko_fault:
START_EXCEPTION(instruction_tlb_miss_htw)
TLB_MISS_PROLOG
+ ld r15,EX_TLB_PACA(r12)
+
/* If we take a recursive fault, the second level handler may need
* to know whether we are handling a data or instruction fault in
* order to get to the right store fault handler. We provide that
@@ -548,7 +555,7 @@ virt_page_table_tlb_miss_whacko_fault:
/* We do the user/kernel test for the PID here along with the RW test
*/
cmpldi cr0,r11,0 /* Check for user region */
- ld r15,PACAPGD(r13) /* Load user pgdir */
+ add r15,r15,r11
beq htw_tlb_miss
/* XXX replace the RMW cycles with immediate loads + writes */
@@ -556,7 +563,6 @@ virt_page_table_tlb_miss_whacko_fault:
cmpldi cr0,r11,8 /* Check for vmalloc region */
rlwinm r10,r10,0,16,1 /* Clear TID */
mtspr SPRN_MAS1,r10
- ld r15,PACA_KERNELPGD(r13) /* Load kernel pgdir */
beq+ htw_tlb_miss
/* We got a crappy address, just fault */
@@ -570,9 +576,8 @@ virt_page_table_tlb_miss_whacko_fault:
* misses. We are entered with:
*
* r16 = virtual page table faulting address
- * r15 = PGD pointer
+ * r15 = pointer to PGD pointer
* r14 = ESR
- * r13 = PACA
* r12 = TLB exception frame in PACA
* r11 = crap (free to use)
* r10 = crap (free to use)
@@ -581,6 +586,8 @@ virt_page_table_tlb_miss_whacko_fault:
* avoid too much complication, it will save/restore things for us
*/
htw_tlb_miss:
+ ld r15,0(r15)
+
/* Search if we already have a TLB entry for that virtual address, and
* if we do, bail out.
*
@@ -692,7 +699,6 @@ htw_tlb_miss_fault:
* r16 = faulting address
* r15 = crap (free to use)
* r14 = ESR (data) or -1 (instruction)
- * r13 = PACA
* r12 = TLB exception frame in PACA
* r11 = crap (free to use)
* r10 = crap (free to use)
@@ -714,7 +720,8 @@ tlb_load_linear:
* we only use 1G pages for now. That might have to be changed in a
* final implementation, especially when dealing with hypervisors
*/
- ld r11,PACATOC(r13)
+ ld r11,EX_TLB_PACA(r12)
+ ld r11,PACATOC(r11)
ld r11,linear_map_top at got(r11)
ld r10,0(r11)
cmpld cr0,r10,r16
--
1.7.4.1
More information about the Linuxppc-dev
mailing list