[RFC] powerpc: e6500 optimised copy_to_user/copy_from_user
Kim Phillips
kim.phillips at freescale.com
Fri Mar 13 09:45:49 AEDT 2015
This mimics commit a66086b8197da8dc83b698642d5947ff850e708d "powerpc:
POWER7 optimised copy_to_user/copy_from_user using VMX", but for
the e6500, or BOOK3S_64. Changes have been made for the smaller
cacheline size (64 bytes on e6500), and data cache block touch (dcbt)
instructions have been rewritten to prefetch 8 lines ahead, based
on preliminary benchmark results and perf -e L1-dcache-prefetches
and misses observations.
We see a gain of 5% in large netperf benchmarks between two T4240s,
both in terms of throughput, and latency. The same netperf
lo(opback) test improves 27%.
Anton's microbenchmark results show a clear linear improvement path
with sizes 32KB and above, where, below that, the additional overhead
over the existing copyuser_64 implementation shows its head: e.g., 6%
for 1448 byte copies. The observed transfer sizes under large,
netperf benchmarks show the TCP stack is invoking copies on the
order of a few 10's of KB, however. 1MB transfers are 30% better-off
on wall clock time.
RFC because of the following known issues:
- unsure if PPC_BOOK3E_64 vs. PPC_BOOK3S_64 build-time switch to
re-target __copy_tofrom_user_vmx is appropriate (ppc64_defconfig
builds fine, however)
- syscalls report deficits when folding vmx_unaligned_copy to a 64B
cacheline (undone for this RFC)
- any consideration for the e5500?
- asm branch label re-enumeration
- ..I'm sure I've missed another couple of things, possibly
including how to fix lower-sized transfer performance
Signed-off-by: Kim Phillips <kim.phillips at freescale.com>
---
applies to Linus' ToT today (commit 09d35919b06),
since Scott's tree seems a bit outdated.
arch/powerpc/include/asm/cputable.h | 2 +-
arch/powerpc/lib/Makefile | 5 +-
arch/powerpc/lib/copyuser_64.S | 2 +-
arch/powerpc/lib/copyuser_e6500.S | 768 ++++++++++++++++++++++++++++++++++++
arch/powerpc/lib/copyuser_power7.S | 2 +-
5 files changed, 775 insertions(+), 4 deletions(-)
create mode 100644 arch/powerpc/lib/copyuser_e6500.S
diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
index 5cf5a6d..66ca94a 100644
--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -391,7 +391,7 @@ extern const char *powerpc_base_platform;
CPU_FTR_L2CSR | CPU_FTR_LWSYNC | CPU_FTR_NOEXECUTE | \
CPU_FTR_DBELL | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
CPU_FTR_DEBUG_LVL_EXC | CPU_FTR_EMB_HV | CPU_FTR_ALTIVEC_COMP | \
- CPU_FTR_CELL_TB_BUG | CPU_FTR_SMT)
+ CPU_FTR_CELL_TB_BUG | CPU_FTR_SMT | CPU_FTR_VMX_COPY)
#define CPU_FTRS_GENERIC_32 (CPU_FTR_COMMON | CPU_FTR_NODSISRALIGN)
/* 64-bit CPUs */
diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
index 7902802..c85e05d 100644
--- a/arch/powerpc/lib/Makefile
+++ b/arch/powerpc/lib/Makefile
@@ -15,12 +15,15 @@ obj-y += string.o alloc.o crtsavres.o ppc_ksyms.o code-patching.o \
obj-$(CONFIG_PPC32) += div64.o copy_32.o
obj64-y += copypage_64.o copyuser_64.o usercopy_64.o mem_64.o hweight_64.o \
- copyuser_power7.o string_64.o copypage_power7.o memcpy_power7.o \
+ string_64.o copypage_power7.o memcpy_power7.o \
memcpy_64.o memcmp_64.o
obj64-$(CONFIG_SMP) += locks.o
obj64-$(CONFIG_ALTIVEC) += vmx-helper.o
+obj64-$(CONFIG_PPC_BOOK3S_64) += copyuser_power7.o
+obj64-$(CONFIG_PPC_BOOK3E_64) += copyuser_e6500.o
+
ifeq ($(CONFIG_GENERIC_CSUM),)
obj-y += checksum_$(CONFIG_WORD_SIZE).o
obj-$(CONFIG_PPC64) += checksum_wrappers_64.o
diff --git a/arch/powerpc/lib/copyuser_64.S b/arch/powerpc/lib/copyuser_64.S
index f09899e..0cf9253 100644
--- a/arch/powerpc/lib/copyuser_64.S
+++ b/arch/powerpc/lib/copyuser_64.S
@@ -22,7 +22,7 @@ _GLOBAL_TOC(__copy_tofrom_user)
BEGIN_FTR_SECTION
nop
FTR_SECTION_ELSE
- b __copy_tofrom_user_power7
+ b __copy_tofrom_user_vmx
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
_GLOBAL(__copy_tofrom_user_base)
/* first check for a whole page copy on a page boundary */
diff --git a/arch/powerpc/lib/copyuser_e6500.S b/arch/powerpc/lib/copyuser_e6500.S
new file mode 100644
index 0000000..efd4251
--- /dev/null
+++ b/arch/powerpc/lib/copyuser_e6500.S
@@ -0,0 +1,768 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright Freescale Semiconductor, 2015
+ *
+ * based on code with:
+ * Copyright (C) IBM Corporation, 2011
+ *
+ * Author: Anton Blanchard <anton at au.ibm.com>
+ */
+#include <asm/ppc_asm.h>
+
+#ifdef __BIG_ENDIAN__
+#define LVS(VRT,RA,RB) lvsl VRT,RA,RB
+#define VPERM(VRT,VRA,VRB,VRC) vperm VRT,VRA,VRB,VRC
+#else
+#define LVS(VRT,RA,RB) lvsr VRT,RA,RB
+#define VPERM(VRT,VRA,VRB,VRC) vperm VRT,VRB,VRA,VRC
+#endif
+
+ .macro err1
+100:
+ .section __ex_table,"a"
+ .align 3
+ .llong 100b,.Ldo_err1
+ .previous
+ .endm
+
+ .macro err2
+200:
+ .section __ex_table,"a"
+ .align 3
+ .llong 200b,.Ldo_err2
+ .previous
+ .endm
+
+#ifdef CONFIG_ALTIVEC
+ .macro err3
+300:
+ .section __ex_table,"a"
+ .align 3
+ .llong 300b,.Ldo_err3
+ .previous
+ .endm
+
+ .macro err4
+400:
+ .section __ex_table,"a"
+ .align 3
+ .llong 400b,.Ldo_err4
+ .previous
+ .endm
+
+
+.Ldo_err4:
+.Ldo_err3:
+ bl exit_vmx_usercopy
+ ld r0,STACKFRAMESIZE+16(r1)
+ mtlr r0
+ b .Lexit
+#endif /* CONFIG_ALTIVEC */
+
+.Ldo_err2:
+ ld r14,STK_REG(R15)(r1)
+ ld r14,STK_REG(R14)(r1)
+.Lexit:
+ addi r1,r1,STACKFRAMESIZE
+.Ldo_err1:
+ ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
+ ld r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
+ ld r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
+ b __copy_tofrom_user_base
+
+
+_GLOBAL(__copy_tofrom_user_vmx)
+ dcbt 0,r4
+ dcbtst 0,r3
+#ifdef CONFIG_ALTIVEC
+ cmpldi r5,64
+ cmpldi cr1,r5,4096
+
+ std r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
+ std r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
+ std r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
+
+ blt .Lshort_copy
+ bgt cr1,.Lvmx_copy
+#else
+ cmpldi r5,64
+
+ std r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
+ std r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
+ std r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
+
+ blt .Lshort_copy
+#endif
+
+.Lnonvmx_copy:
+ /* Get the source 8B aligned */
+ neg r6,r4
+ mtocrf 0x01,r6
+ clrldi r6,r6,(64-3)
+
+ bf cr7*4+3,1f
+err1; lbz r0,0(r4)
+ addi r4,r4,1
+err1; stb r0,0(r3)
+ addi r3,r3,1
+
+1: bf cr7*4+2,2f
+err1; lhz r0,0(r4)
+ addi r4,r4,2
+err1; sth r0,0(r3)
+ addi r3,r3,2
+
+2: bf cr7*4+1,3f
+err1; lwz r0,0(r4)
+ addi r4,r4,4
+err1; stw r0,0(r3)
+ addi r3,r3,4
+
+3: sub r5,r5,r6
+ cmpldi r5,64
+ blt 5f
+
+ mflr r0
+ stdu r1,-STACKFRAMESIZE(r1)
+ std r14,STK_REG(R14)(r1)
+ std r15,STK_REG(R15)(r1)
+ std r0,STACKFRAMESIZE+16(r1)
+
+#define LINES_AHEAD 8
+ /*
+ * We prefetch both the source and destination using regular touch
+ * instructions.
+ */
+ clrrdi r6,r4,6
+ clrrdi r9,r3,6
+
+ srdi r7,r5,6 /* length in cachelines, capped at LINES_AHEAD*/
+ cmpldi r7,LINES_AHEAD
+ ble 111f /* continue if cachelines <= LINES_AHEAD */
+ li r7,LINES_AHEAD /* otherwise, r7 <- LINES_AHEAD */
+111: lis r0,0
+
+ mtctr r7
+222:
+ addi r6,r6,64
+ addi r9,r9,64
+
+ dcbt r0,r6 /* addr from */
+ dcbtst r0,r9 /* addr to */
+
+ bdnz 222b
+
+ li r14,64*LINES_AHEAD
+
+ srdi r15,r5,6 /* length in cachelines */
+ cmpldi r15,LINES_AHEAD
+ ble 442f /* don't prefetch if cachelines <= LINES_AHEAD */
+ subi r15,r15,LINES_AHEAD /* otherwise, r15 <- r15 - LINES_AHEAD */
+ mtctr r15
+
+ /* Now do cacheline (64B) sized loads and stores WITH prefetches. */
+ .align 5
+4:
+err2; ld r0,0(r4)
+err2; ld r6,8(r4)
+err2; ld r7,16(r4)
+err2; ld r8,24(r4)
+err2; ld r9,32(r4)
+err2; ld r10,40(r4)
+err2; ld r11,48(r4)
+err2; ld r12,56(r4)
+
+ addi r4,r4,64
+ dcbt r14,r4
+err2; std r0,0(r3)
+err2; std r6,8(r3)
+err2; std r7,16(r3)
+err2; std r8,24(r3)
+err2; std r9,32(r3)
+err2; std r10,40(r3)
+err2; std r11,48(r3)
+err2; std r12,56(r3)
+
+ addi r3,r3,64
+ dcbtst r14,r3
+ bdnz 4b
+
+ srdi r7,r5,6 /* length in cachelines */
+ subf r15,r15,r7 /* r15 = r7 - r15 */
+442:
+ mtctr r15
+
+ /* remainder cacheline (64B) sized loads and stores WITHOUT prefetches*/
+ .align 5
+444:
+err2; ld r0,0(r4)
+err2; ld r6,8(r4)
+err2; ld r7,16(r4)
+err2; ld r8,24(r4)
+err2; ld r9,32(r4)
+err2; ld r10,40(r4)
+err2; ld r11,48(r4)
+err2; ld r12,56(r4)
+
+ addi r4,r4,64
+err2; std r0,0(r3)
+err2; std r6,8(r3)
+err2; std r7,16(r3)
+err2; std r8,24(r3)
+err2; std r9,32(r3)
+err2; std r10,40(r3)
+err2; std r11,48(r3)
+err2; std r12,56(r3)
+
+ addi r3,r3,64
+ bdnz 444b
+
+ clrldi r5,r5,(64-6)
+
+ ld r14,STK_REG(R14)(r1)
+ ld r15,STK_REG(R15)(r1)
+ addi r1,r1,STACKFRAMESIZE
+
+ /* Up to 63B to go */
+.Lshort_copy:
+5: srdi r6,r5,4
+ mtocrf 0x01,r6
+
+7: bf cr7*4+2,8f
+err1; ld r0,0(r4)
+err1; ld r6,8(r4)
+err1; ld r7,16(r4)
+err1; ld r8,24(r4)
+ addi r4,r4,32
+err1; std r0,0(r3)
+err1; std r6,8(r3)
+err1; std r7,16(r3)
+err1; std r8,24(r3)
+ addi r3,r3,32
+
+ /* Up to 31B to go */
+8: bf cr7*4+3,9f
+err1; ld r0,0(r4)
+err1; ld r6,8(r4)
+ addi r4,r4,16
+err1; std r0,0(r3)
+err1; std r6,8(r3)
+ addi r3,r3,16
+
+9: clrldi r5,r5,(64-4)
+
+ /* Up to 15B to go */
+ mtocrf 0x01,r5
+ bf cr7*4+0,12f
+err1; lwz r0,0(r4) /* Less chance of a reject with word ops */
+err1; lwz r6,4(r4)
+ addi r4,r4,8
+err1; stw r0,0(r3)
+err1; stw r6,4(r3)
+ addi r3,r3,8
+
+12: bf cr7*4+1,13f
+err1; lwz r0,0(r4)
+ addi r4,r4,4
+err1; stw r0,0(r3)
+ addi r3,r3,4
+
+13: bf cr7*4+2,14f
+err1; lhz r0,0(r4)
+ addi r4,r4,2
+err1; sth r0,0(r3)
+ addi r3,r3,2
+
+14: bf cr7*4+3,15f
+err1; lbz r0,0(r4)
+err1; stb r0,0(r3)
+
+15: li r3,0
+ blr
+
+.Lunwind_stack_nonvmx_copy:
+ addi r1,r1,STACKFRAMESIZE
+ b .Lnonvmx_copy
+
+#ifdef CONFIG_ALTIVEC
+.Lvmx_copy:
+ mflr r0
+ std r0,16(r1)
+ stdu r1,-STACKFRAMESIZE(r1)
+ bl enter_vmx_usercopy
+ cmpwi cr1,r3,0
+ ld r0,STACKFRAMESIZE+16(r1)
+ ld r3,STK_REG(R31)(r1)
+ ld r4,STK_REG(R30)(r1)
+ ld r5,STK_REG(R29)(r1)
+ mtlr r0
+
+ /*
+ * We prefetch both the source and destination using regular touch
+ * instructions.
+ */
+ clrrdi r6,r4,6
+ clrrdi r9,r3,6
+
+ srdi r7,r5,6 /* length in cachelines, capped at LINES_AHEAD*/
+ cmpldi r7,LINES_AHEAD
+ ble 333f
+ li r7,LINES_AHEAD
+333: lis r0,0
+
+ mtctr r7
+444:
+ addi r6,r6,64
+ addi r9,r9,64
+
+ dcbt r0,r6 /* addr from */
+ dcbtst r0,r9 /* addr to */
+
+ bdnz 444b
+
+ beq cr1,.Lunwind_stack_nonvmx_copy
+
+ /*
+ * If source and destination are not relatively aligned we use a
+ * slower permute loop.
+ */
+ xor r6,r4,r3
+ rldicl. r6,r6,0,(64-4)
+ bne .Lvmx_unaligned_copy
+
+ /* Get the destination 16B aligned */
+ neg r6,r3
+ mtocrf 0x01,r6
+ clrldi r6,r6,(64-4)
+
+ bf cr7*4+3,1f
+err3; lbz r0,0(r4)
+ addi r4,r4,1
+err3; stb r0,0(r3)
+ addi r3,r3,1
+
+1: bf cr7*4+2,2f
+err3; lhz r0,0(r4)
+ addi r4,r4,2
+err3; sth r0,0(r3)
+ addi r3,r3,2
+
+2: bf cr7*4+1,3f
+err3; lwz r0,0(r4)
+ addi r4,r4,4
+err3; stw r0,0(r3)
+ addi r3,r3,4
+
+3: bf cr7*4+0,4f
+err3; ld r0,0(r4)
+ addi r4,r4,8
+err3; std r0,0(r3)
+ addi r3,r3,8
+
+4: sub r5,r5,r6
+
+ /* Get the destination 64B aligned */
+ neg r6,r3
+ srdi r7,r6,4
+ mtocrf 0x01,r7
+ clrldi r6,r6,(64-6)
+
+ li r9,16
+ li r10,32
+ li r11,48
+
+ bf cr7*4+3,5f
+err3; lvx vr1,r0,r4
+ addi r4,r4,16
+err3; stvx vr1,r0,r3
+ addi r3,r3,16
+
+5: bf cr7*4+2,7f
+err3; lvx vr1,r0,r4
+err3; lvx vr0,r4,r9
+ addi r4,r4,32
+err3; stvx vr1,r0,r3
+err3; stvx vr0,r3,r9
+ addi r3,r3,32
+
+7: sub r5,r5,r6 /* r5 <- r6 - r5 */
+ srdi r6,r5,6
+
+ cmpldi r6,LINES_AHEAD
+ ble 882f /* branch if cachelines <= LINES_AHEAD */
+ subi r6,r6,LINES_AHEAD /* otherwise, r6 <- r6 - LINES_AHEAD */
+
+ li r12,64*LINES_AHEAD
+
+ mtctr r6
+
+ /*
+ * Now do cacheline (64B) sized loads and stores WITH prefetches.
+ * By this stage the cacheline stores are also cacheline aligned.
+ */
+ .align 5
+8:
+err4; lvx vr7,r0,r4
+err4; lvx vr6,r4,r9
+err4; lvx vr5,r4,r10
+err4; lvx vr4,r4,r11
+
+ addi r4,r4,64
+
+ dcbt r12,r4
+
+err4; stvx vr7,r0,r3
+err4; stvx vr6,r3,r9
+err4; stvx vr5,r3,r10
+err4; stvx vr4,r3,r11
+
+ addi r3,r3,64
+
+ dcbtst r12,r3
+
+ bdnz 8b
+
+ srdi r7,r5,6 /* length in cachelines */
+ subf r6,r6,r7 /* r6 = r7 - r6 */
+882:
+ mtctr r6
+ /* remainder cacheline (64B) sized loads and stores WITHOUT prefetches. */
+ .align 5
+88:
+err4; lvx vr7,r0,r4
+err4; lvx vr6,r4,r9
+err4; lvx vr5,r4,r10
+err4; lvx vr4,r4,r11
+
+ addi r4,r4,64
+
+err4; stvx vr7,r0,r3
+err4; stvx vr6,r3,r9
+err4; stvx vr5,r3,r10
+err4; stvx vr4,r3,r11
+
+ addi r3,r3,64
+
+ bdnz 88b
+
+ /* Up to 63B to go */
+ clrldi r5,r5,(64-6)
+ srdi r6,r5,4
+ mtocrf 0x01,r6
+
+9: bf cr7*4+2,10f
+err3; lvx vr1,r0,r4
+err3; lvx vr0,r4,r9
+ addi r4,r4,32
+err3; stvx vr1,r0,r3
+err3; stvx vr0,r3,r9
+ addi r3,r3,32
+
+10: bf cr7*4+3,11f
+err3; lvx vr1,r0,r4
+ addi r4,r4,16
+err3; stvx vr1,r0,r3
+ addi r3,r3,16
+
+ /* Up to 15B to go */
+11: clrldi r5,r5,(64-4)
+ mtocrf 0x01,r5
+ bf cr7*4+0,12f
+err3; ld r0,0(r4)
+ addi r4,r4,8
+err3; std r0,0(r3)
+ addi r3,r3,8
+
+12: bf cr7*4+1,13f
+err3; lwz r0,0(r4)
+ addi r4,r4,4
+err3; stw r0,0(r3)
+ addi r3,r3,4
+
+13: bf cr7*4+2,14f
+err3; lhz r0,0(r4)
+ addi r4,r4,2
+err3; sth r0,0(r3)
+ addi r3,r3,2
+
+14: bf cr7*4+3,15f
+err3; lbz r0,0(r4)
+err3; stb r0,0(r3)
+
+15: addi r1,r1,STACKFRAMESIZE
+ b exit_vmx_usercopy /* tail call optimise */
+
+.Lvmx_unaligned_copy:
+ /* Get the destination 16B aligned */
+ neg r6,r3
+ mtocrf 0x01,r6
+ clrldi r6,r6,(64-4)
+
+ bf cr7*4+3,1f
+err3; lbz r0,0(r4)
+ addi r4,r4,1
+err3; stb r0,0(r3)
+ addi r3,r3,1
+
+1: bf cr7*4+2,2f
+err3; lhz r0,0(r4)
+ addi r4,r4,2
+err3; sth r0,0(r3)
+ addi r3,r3,2
+
+2: bf cr7*4+1,3f
+err3; lwz r0,0(r4)
+ addi r4,r4,4
+err3; stw r0,0(r3)
+ addi r3,r3,4
+
+3: bf cr7*4+0,4f
+err3; lwz r0,0(r4) /* Less chance of a reject with word ops */
+err3; lwz r7,4(r4)
+ addi r4,r4,8
+err3; stw r0,0(r3)
+err3; stw r7,4(r3)
+ addi r3,r3,8
+
+4: sub r5,r5,r6
+
+ /* Get the destination 128B aligned */
+ neg r6,r3
+ srdi r7,r6,4
+ mtocrf 0x01,r7
+ clrldi r6,r6,(64-7)
+
+ li r9,16
+ li r10,32
+ li r11,48
+
+ LVS(vr16,0,r4) /* Setup permute control vector */
+err3; lvx vr0,0,r4
+ addi r4,r4,16
+
+ bf cr7*4+3,5f
+err3; lvx vr1,r0,r4
+ VPERM(vr8,vr0,vr1,vr16)
+ addi r4,r4,16
+err3; stvx vr8,r0,r3
+ addi r3,r3,16
+ vor vr0,vr1,vr1
+
+5: bf cr7*4+2,6f
+err3; lvx vr1,r0,r4
+ VPERM(vr8,vr0,vr1,vr16)
+err3; lvx vr0,r4,r9
+ VPERM(vr9,vr1,vr0,vr16)
+ addi r4,r4,32
+err3; stvx vr8,r0,r3
+err3; stvx vr9,r3,r9
+ addi r3,r3,32
+
+6: bf cr7*4+1,7f
+err3; lvx vr3,r0,r4
+ VPERM(vr8,vr0,vr3,vr16)
+err3; lvx vr2,r4,r9
+ VPERM(vr9,vr3,vr2,vr16)
+err3; lvx vr1,r4,r10
+ VPERM(vr10,vr2,vr1,vr16)
+err3; lvx vr0,r4,r11
+ VPERM(vr11,vr1,vr0,vr16)
+ addi r4,r4,64
+err3; stvx vr8,r0,r3
+err3; stvx vr9,r3,r9
+err3; stvx vr10,r3,r10
+err3; stvx vr11,r3,r11
+ addi r3,r3,64
+
+7: sub r5,r5,r6
+ srdi r6,r5,7
+
+ cmpldi r6,LINES_AHEAD
+ ble 982f /* branch if cachelines <= LINES_AHEAD */
+ subi r6,r6,LINES_AHEAD /* otherwise, r6 <- r6 - LINES_AHEAD */
+
+ std r14,STK_REG(R14)(r1)
+ std r15,STK_REG(R15)(r1)
+ std r16,STK_REG(R16)(r1)
+
+ li r12,64
+ li r14,80
+ li r15,96
+ li r16,112
+
+ li r8,64*LINES_AHEAD
+
+ mtctr r6
+
+ /*
+ * Now do cacheline (128B) sized loads and stores WITH prefetches.
+ * By this stage the cacheline stores are also cacheline aligned.
+ */
+ .align 5
+8:
+err4; lvx vr7,r0,r4
+ VPERM(vr8,vr0,vr7,vr16)
+err4; lvx vr6,r4,r9
+ VPERM(vr9,vr7,vr6,vr16)
+err4; lvx vr5,r4,r10
+ VPERM(vr10,vr6,vr5,vr16)
+err4; lvx vr4,r4,r11
+ VPERM(vr11,vr5,vr4,vr16)
+err4; lvx vr3,r4,r12
+ VPERM(vr12,vr4,vr3,vr16)
+err4; lvx vr2,r4,r14
+ VPERM(vr13,vr3,vr2,vr16)
+err4; lvx vr1,r4,r15
+ VPERM(vr14,vr2,vr1,vr16)
+err4; lvx vr0,r4,r16
+ VPERM(vr15,vr1,vr0,vr16)
+
+ addi r4,r4,64
+ dcbt r8,r4
+ addi r4,r4,64
+ dcbt r8,r4
+
+err4; stvx vr8,r0,r3
+err4; stvx vr9,r3,r9
+err4; stvx vr10,r3,r10
+err4; stvx vr11,r3,r11
+err4; stvx vr12,r3,r12
+err4; stvx vr13,r3,r14
+err4; stvx vr14,r3,r15
+err4; stvx vr15,r3,r16
+
+ addi r3,r3,64
+ dcbtst r8,r3
+ addi r3,r3,64
+ dcbtst r8,r3
+
+ bdnz 8b
+
+ srdi r7,r5,7 /* length in double-cachelines */
+ subf r6,r6,r7 /* r6 = r7 - r6 */
+982:
+ mtctr r6
+ /*
+ * remainder double-cacheline (128B) sized loads and stores
+ * WITHOUT prefetches.
+ */
+ .align 5
+98:
+err4; lvx vr7,r0,r4
+ VPERM(vr8,vr0,vr7,vr16)
+err4; lvx vr6,r4,r9
+ VPERM(vr9,vr7,vr6,vr16)
+err4; lvx vr5,r4,r10
+ VPERM(vr10,vr6,vr5,vr16)
+err4; lvx vr4,r4,r11
+ VPERM(vr11,vr5,vr4,vr16)
+err4; lvx vr3,r4,r12
+ VPERM(vr12,vr4,vr3,vr16)
+err4; lvx vr2,r4,r14
+ VPERM(vr13,vr3,vr2,vr16)
+err4; lvx vr1,r4,r15
+ VPERM(vr14,vr2,vr1,vr16)
+err4; lvx vr0,r4,r16
+ VPERM(vr15,vr1,vr0,vr16)
+
+ addi r4,r4,128
+
+err4; stvx vr8,r0,r3
+err4; stvx vr9,r3,r9
+err4; stvx vr10,r3,r10
+err4; stvx vr11,r3,r11
+err4; stvx vr12,r3,r12
+err4; stvx vr13,r3,r14
+err4; stvx vr14,r3,r15
+err4; stvx vr15,r3,r16
+
+ addi r3,r3,128
+
+ bdnz 98b
+
+ ld r14,STK_REG(R14)(r1)
+ ld r15,STK_REG(R15)(r1)
+ ld r16,STK_REG(R16)(r1)
+
+ /* Up to 127B to go */
+ clrldi r5,r5,(64-7)
+ srdi r6,r5,4
+ mtocrf 0x01,r6
+
+ bf cr7*4+1,9f
+err3; lvx vr3,r0,r4
+ VPERM(vr8,vr0,vr3,vr16)
+err3; lvx vr2,r4,r9
+ VPERM(vr9,vr3,vr2,vr16)
+err3; lvx vr1,r4,r10
+ VPERM(vr10,vr2,vr1,vr16)
+err3; lvx vr0,r4,r11
+ VPERM(vr11,vr1,vr0,vr16)
+ addi r4,r4,64
+err3; stvx vr8,r0,r3
+err3; stvx vr9,r3,r9
+err3; stvx vr10,r3,r10
+err3; stvx vr11,r3,r11
+ addi r3,r3,64
+
+9: bf cr7*4+2,10f
+err3; lvx vr1,r0,r4
+ VPERM(vr8,vr0,vr1,vr16)
+err3; lvx vr0,r4,r9
+ VPERM(vr9,vr1,vr0,vr16)
+ addi r4,r4,32
+err3; stvx vr8,r0,r3
+err3; stvx vr9,r3,r9
+ addi r3,r3,32
+
+10: bf cr7*4+3,11f
+err3; lvx vr1,r0,r4
+ VPERM(vr8,vr0,vr1,vr16)
+ addi r4,r4,16
+err3; stvx vr8,r0,r3
+ addi r3,r3,16
+
+ /* Up to 15B to go */
+11: clrldi r5,r5,(64-4)
+ addi r4,r4,-16 /* Unwind the +16 load offset */
+ mtocrf 0x01,r5
+ bf cr7*4+0,12f
+err3; lwz r0,0(r4) /* Less chance of a reject with word ops */
+err3; lwz r6,4(r4)
+ addi r4,r4,8
+err3; stw r0,0(r3)
+err3; stw r6,4(r3)
+ addi r3,r3,8
+
+12: bf cr7*4+1,13f
+err3; lwz r0,0(r4)
+ addi r4,r4,4
+err3; stw r0,0(r3)
+ addi r3,r3,4
+
+13: bf cr7*4+2,14f
+err3; lhz r0,0(r4)
+ addi r4,r4,2
+err3; sth r0,0(r3)
+ addi r3,r3,2
+
+14: bf cr7*4+3,15f
+err3; lbz r0,0(r4)
+err3; stb r0,0(r3)
+
+15: addi r1,r1,STACKFRAMESIZE
+ b exit_vmx_usercopy /* tail call optimise */
+#endif /* CONFIG_ALTIVEC */
diff --git a/arch/powerpc/lib/copyuser_power7.S b/arch/powerpc/lib/copyuser_power7.S
index 92ee840..258e6fc 100644
--- a/arch/powerpc/lib/copyuser_power7.S
+++ b/arch/powerpc/lib/copyuser_power7.S
@@ -91,7 +91,7 @@
b __copy_tofrom_user_base
-_GLOBAL(__copy_tofrom_user_power7)
+_GLOBAL(__copy_tofrom_user_vmx)
#ifdef CONFIG_ALTIVEC
cmpldi r5,16
cmpldi cr1,r5,4096
--
2.3.2
More information about the Linuxppc-dev
mailing list