Merge master.kernel.org:/pub/scm/linux/kernel/git/wim/linux-2.6-watchdog

[linux-2.6-omap-h63xx.git] / arch / x86_64 / lib / copy_page.S

diff --git a/arch/x86_64/lib/copy_page.S b/arch/x86_64/lib/copy_page.S
index 621a19769406c2f25dbd4b04bc578027e6c03958..8fa19d96a7eefde6b7d591fba46705486055818f 100644 (file)
--- a/arch/x86_64/lib/copy_page.S
+++ b/arch/x86_64/lib/copy_page.S
@@ -8,7 +8,94 @@
        .globl copy_page
        .p2align 4
 copy_page:
+       subq    $3*8,%rsp
+       movq    %rbx,(%rsp)
+       movq    %r12,1*8(%rsp)
+       movq    %r13,2*8(%rsp)
+
+       movl    $(4096/64)-5,%ecx
+       .p2align 4
+.Loop64:
+       dec     %rcx
+
+       movq        (%rsi), %rax
+       movq      8 (%rsi), %rbx
+       movq     16 (%rsi), %rdx
+       movq     24 (%rsi), %r8
+       movq     32 (%rsi), %r9
+       movq     40 (%rsi), %r10
+       movq     48 (%rsi), %r11
+       movq     56 (%rsi), %r12
+
+       prefetcht0 5*64(%rsi)
+
+       movq     %rax,    (%rdi)
+       movq     %rbx,  8 (%rdi)
+       movq     %rdx, 16 (%rdi)
+       movq     %r8,  24 (%rdi)
+       movq     %r9,  32 (%rdi)
+       movq     %r10, 40 (%rdi)
+       movq     %r11, 48 (%rdi)
+       movq     %r12, 56 (%rdi)
+
+       leaq    64 (%rsi), %rsi
+       leaq    64 (%rdi), %rdi
+
+       jnz     .Loop64
+
+       movl    $5,%ecx
+       .p2align 4
+.Loop2:
+       decl   %ecx
+
+       movq        (%rsi), %rax
+       movq      8 (%rsi), %rbx
+       movq     16 (%rsi), %rdx
+       movq     24 (%rsi), %r8
+       movq     32 (%rsi), %r9
+       movq     40 (%rsi), %r10
+       movq     48 (%rsi), %r11
+       movq     56 (%rsi), %r12
+
+       movq     %rax,    (%rdi)
+       movq     %rbx,  8 (%rdi)
+       movq     %rdx, 16 (%rdi)
+       movq     %r8,  24 (%rdi)
+       movq     %r9,  32 (%rdi)
+       movq     %r10, 40 (%rdi)
+       movq     %r11, 48 (%rdi)
+       movq     %r12, 56 (%rdi)
+
+       leaq    64(%rdi),%rdi
+       leaq    64(%rsi),%rsi
+
+       jnz     .Loop2
+
+       movq    (%rsp),%rbx
+       movq    1*8(%rsp),%r12
+       movq    2*8(%rsp),%r13
+       addq    $3*8,%rsp
+       ret
+
+       /* Some CPUs run faster using the string copy instructions.
+          It is also a lot simpler. Use this when possible */
+
+#include <asm/cpufeature.h>
+
+       .section .altinstructions,"a"
+       .align 8
+       .quad  copy_page
+       .quad  copy_page_c
+       .byte  X86_FEATURE_REP_GOOD
+       .byte  copy_page_c_end-copy_page_c
+       .byte  copy_page_c_end-copy_page_c
+       .previous
+
+       .section .altinstr_replacement,"ax"
+copy_page_c:
        movl $4096/8,%ecx
        rep 
        movsq 
        ret
+copy_page_c_end:
+       .previous