leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x86: fix copy_user on x86

Browse Source 
Switch copy_user_generic_string(), copy_user_generic_unrolled() and
__copy_user_nocache() from custom tail handlers to generic
copy_user_tail_handle().

Signed-off-by: Vitaly Mayatskikh <v.mayatskih@gmail.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Vitaly Mayatskikh 2008-07-02 15:53:13 +02:00 committed by Ingo Molnar
parent 1129585a08
commit ad2fc2cd92
2 changed files with 284 additions and 444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

411
arch/x86/lib/copy_user_64.S
View File

@ -1,4 +1,6 @@
/* Copyright 2002 Andi Kleen, SuSE Labs. /*
* Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
* Copyright 2002 Andi Kleen, SuSE Labs.
* Subject to the GNU Public License v2. * Subject to the GNU Public License v2.
* *
* Functions to copy from and to user space. * Functions to copy from and to user space.
@ -33,6 +35,35 @@
.previous .previous
.endm .endm
.macro ALIGN_DESTINATION
#ifdef FIX_ALIGNMENT
/* check for bad alignment of destination */
movl %edi,%ecx
andl $7,%ecx
jz 102f /* already aligned */
subl $8,%ecx
negl %ecx
subl %ecx,%edx
100: movb (%rsi),%al
101: movb %al,(%rdi)
incq %rsi
incq %rdi
decl %ecx
jnz 100b
102:
.section .fixup,"ax"
103: addl %r8d,%edx /* ecx is zerorest also */
jmp copy_user_handle_tail
.previous
.section __ex_table,"a"
.align 8
.quad 100b,103b
.quad 101b,103b
.previous
#endif
.endm
/* Standard copy_to_user with segment limit checking */ /* Standard copy_to_user with segment limit checking */
ENTRY(copy_to_user) ENTRY(copy_to_user)
CFI_STARTPROC CFI_STARTPROC
@ -42,19 +73,6 @@ ENTRY(copy_to_user)
jc bad_to_user jc bad_to_user
cmpq TI_addr_limit(%rax),%rcx cmpq TI_addr_limit(%rax),%rcx
jae bad_to_user jae bad_to_user
xorl %eax,%eax /* clear zero flag */
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
CFI_ENDPROC
ENTRY(copy_user_generic)
CFI_STARTPROC
movl $1,%ecx /* set zero flag */
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
CFI_ENDPROC
ENTRY(__copy_from_user_inatomic)
CFI_STARTPROC
xorl %ecx,%ecx /* clear zero flag */
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
CFI_ENDPROC CFI_ENDPROC
@ -67,13 +85,25 @@ ENTRY(copy_from_user)
jc bad_from_user jc bad_from_user
cmpq TI_addr_limit(%rax),%rcx cmpq TI_addr_limit(%rax),%rcx
jae bad_from_user jae bad_from_user
movl $1,%ecx /* set zero flag */
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
CFI_ENDPROC CFI_ENDPROC
ENDPROC(copy_from_user) ENDPROC(copy_from_user)
ENTRY(copy_user_generic)
CFI_STARTPROC
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
CFI_ENDPROC
ENDPROC(copy_user_generic)
ENTRY(__copy_from_user_inatomic)
CFI_STARTPROC
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
CFI_ENDPROC
ENDPROC(__copy_from_user_inatomic)
.section .fixup,"ax" .section .fixup,"ax"
/* must zero dest */ /* must zero dest */
ENTRY(bad_from_user)
bad_from_user: bad_from_user:
CFI_STARTPROC CFI_STARTPROC
movl %edx,%ecx movl %edx,%ecx
@ -84,219 +114,111 @@ bad_to_user:
movl %edx,%eax movl %edx,%eax
ret ret
CFI_ENDPROC CFI_ENDPROC
END(bad_from_user) ENDPROC(bad_from_user)
.previous .previous
/* /*
* copy_user_generic_unrolled - memory copy with exception handling. * copy_user_generic_unrolled - memory copy with exception handling.
* This version is for CPUs like P4 that don't have efficient micro code for rep movsq * This version is for CPUs like P4 that don't have efficient micro
* code for rep movsq
* *
* Input: * Input:
* rdi destination * rdi destination
* rsi source * rsi source
* rdx count * rdx count
* ecx zero flag -- if true zero destination on error
*
* Output:
* eax uncopied bytes or 0 if successful.
*/
ENTRY(copy_user_generic_unrolled)
CFI_STARTPROC
pushq %rbx
CFI_ADJUST_CFA_OFFSET 8
CFI_REL_OFFSET rbx, 0
pushq %rcx
CFI_ADJUST_CFA_OFFSET 8
CFI_REL_OFFSET rcx, 0
xorl %eax,%eax /*zero for the exception handler */
#ifdef FIX_ALIGNMENT
/* check for bad alignment of destination */
movl %edi,%ecx
andl $7,%ecx
jnz .Lbad_alignment
.Lafter_bad_alignment:
#endif
movq %rdx,%rcx
movl $64,%ebx
shrq $6,%rdx
decq %rdx
js .Lhandle_tail
.p2align 4
.Lloop:
.Ls1: movq (%rsi),%r11
.Ls2: movq 1*8(%rsi),%r8
.Ls3: movq 2*8(%rsi),%r9
.Ls4: movq 3*8(%rsi),%r10
.Ld1: movq %r11,(%rdi)
.Ld2: movq %r8,1*8(%rdi)
.Ld3: movq %r9,2*8(%rdi)
.Ld4: movq %r10,3*8(%rdi)
.Ls5: movq 4*8(%rsi),%r11
.Ls6: movq 5*8(%rsi),%r8
.Ls7: movq 6*8(%rsi),%r9
.Ls8: movq 7*8(%rsi),%r10
.Ld5: movq %r11,4*8(%rdi)
.Ld6: movq %r8,5*8(%rdi)
.Ld7: movq %r9,6*8(%rdi)
.Ld8: movq %r10,7*8(%rdi)
decq %rdx
leaq 64(%rsi),%rsi
leaq 64(%rdi),%rdi
jns .Lloop
.p2align 4
.Lhandle_tail:
movl %ecx,%edx
andl $63,%ecx
shrl $3,%ecx
jz .Lhandle_7
movl $8,%ebx
.p2align 4
.Lloop_8:
.Ls9: movq (%rsi),%r8
.Ld9: movq %r8,(%rdi)
decl %ecx
leaq 8(%rdi),%rdi
leaq 8(%rsi),%rsi
jnz .Lloop_8
.Lhandle_7:
movl %edx,%ecx
andl $7,%ecx
jz .Lende
.p2align 4
.Lloop_1:
.Ls10: movb (%rsi),%bl
.Ld10: movb %bl,(%rdi)
incq %rdi
incq %rsi
decl %ecx
jnz .Lloop_1
CFI_REMEMBER_STATE
.Lende:
popq %rcx
CFI_ADJUST_CFA_OFFSET -8
CFI_RESTORE rcx
popq %rbx
CFI_ADJUST_CFA_OFFSET -8
CFI_RESTORE rbx
ret
CFI_RESTORE_STATE
#ifdef FIX_ALIGNMENT
/* align destination */
.p2align 4
.Lbad_alignment:
movl $8,%r9d
subl %ecx,%r9d
movl %r9d,%ecx
cmpq %r9,%rdx
jz .Lhandle_7
js .Lhandle_7
.Lalign_1:
.Ls11: movb (%rsi),%bl
.Ld11: movb %bl,(%rdi)
incq %rsi
incq %rdi
decl %ecx
jnz .Lalign_1
subq %r9,%rdx
jmp .Lafter_bad_alignment
#endif
/* table sorted by exception address */
.section __ex_table,"a"
.align 8
.quad .Ls1,.Ls1e /* Ls1-Ls4 have copied zero bytes */
.quad .Ls2,.Ls1e
.quad .Ls3,.Ls1e
.quad .Ls4,.Ls1e
.quad .Ld1,.Ls1e /* Ld1-Ld4 have copied 0-24 bytes */
.quad .Ld2,.Ls2e
.quad .Ld3,.Ls3e
.quad .Ld4,.Ls4e
.quad .Ls5,.Ls5e /* Ls5-Ls8 have copied 32 bytes */
.quad .Ls6,.Ls5e
.quad .Ls7,.Ls5e
.quad .Ls8,.Ls5e
.quad .Ld5,.Ls5e /* Ld5-Ld8 have copied 32-56 bytes */
.quad .Ld6,.Ls6e
.quad .Ld7,.Ls7e
.quad .Ld8,.Ls8e
.quad .Ls9,.Le_quad
.quad .Ld9,.Le_quad
.quad .Ls10,.Le_byte
.quad .Ld10,.Le_byte
#ifdef FIX_ALIGNMENT
.quad .Ls11,.Lzero_rest
.quad .Ld11,.Lzero_rest
#endif
.quad .Le5,.Le_zero
.previous
/* eax: zero, ebx: 64 */
.Ls1e: addl $8,%eax /* eax is bytes left uncopied within the loop (Ls1e: 64 .. Ls8e: 8) */
.Ls2e: addl $8,%eax
.Ls3e: addl $8,%eax
.Ls4e: addl $8,%eax
.Ls5e: addl $8,%eax
.Ls6e: addl $8,%eax
.Ls7e: addl $8,%eax
.Ls8e: addl $8,%eax
addq %rbx,%rdi /* +64 */
subq %rax,%rdi /* correct destination with computed offset */
shlq $6,%rdx /* loop counter * 64 (stride length) */
addq %rax,%rdx /* add offset to loopcnt */
andl $63,%ecx /* remaining bytes */
addq %rcx,%rdx /* add them */
jmp .Lzero_rest
/* exception on quad word loop in tail handling */
/* ecx: loopcnt/8, %edx: length, rdi: correct */
.Le_quad:
shll $3,%ecx
andl $7,%edx
addl %ecx,%edx
/* edx: bytes to zero, rdi: dest, eax:zero */
.Lzero_rest:
cmpl $0,(%rsp)
jz .Le_zero
movq %rdx,%rcx
.Le_byte:
xorl %eax,%eax
.Le5: rep
stosb
/* when there is another exception while zeroing the rest just return */
.Le_zero:
movq %rdx,%rax
jmp .Lende
CFI_ENDPROC
ENDPROC(copy_user_generic)
/* Some CPUs run faster using the string copy instructions.
This is also a lot simpler. Use them when possible.
Patch in jmps to this code instead of copying it fully
to avoid unwanted aliasing in the exception tables. */
/* rdi destination
* rsi source
* rdx count
* ecx zero flag
* *
* Output: * Output:
* eax uncopied bytes or 0 if successfull. * eax uncopied bytes or 0 if successfull.
*/
ENTRY(copy_user_generic_unrolled)
CFI_STARTPROC
cmpl $8,%edx
jb 20f /* less then 8 bytes, go to byte copy loop */
ALIGN_DESTINATION
movl %edx,%ecx
andl $63,%edx
shrl $6,%ecx
jz 17f
1: movq (%rsi),%r8
2: movq 1*8(%rsi),%r9
3: movq 2*8(%rsi),%r10
4: movq 3*8(%rsi),%r11
5: movq %r8,(%rdi)
6: movq %r9,1*8(%rdi)
7: movq %r10,2*8(%rdi)
8: movq %r11,3*8(%rdi)
9: movq 4*8(%rsi),%r8
10: movq 5*8(%rsi),%r9
11: movq 6*8(%rsi),%r10
12: movq 7*8(%rsi),%r11
13: movq %r8,4*8(%rdi)
14: movq %r9,5*8(%rdi)
15: movq %r10,6*8(%rdi)
16: movq %r11,7*8(%rdi)
leaq 64(%rsi),%rsi
leaq 64(%rdi),%rdi
decl %ecx
jnz 1b
17: movl %edx,%ecx
andl $7,%edx
shrl $3,%ecx
jz 20f
18: movq (%rsi),%r8
19: movq %r8,(%rdi)
leaq 8(%rsi),%rsi
leaq 8(%rdi),%rdi
decl %ecx
jnz 18b
20: andl %edx,%edx
jz 23f
movl %edx,%ecx
21: movb (%rsi),%al
22: movb %al,(%rdi)
incq %rsi
incq %rdi
decl %ecx
jnz 21b
23: xor %eax,%eax
ret
.section .fixup,"ax"
30: shll $6,%ecx
addl %ecx,%edx
jmp 60f
40: leal (%edx,%ecx,8),%edx
jmp 60f
50: movl %ecx,%edx
60: jmp copy_user_handle_tail /* ecx is zerorest also */
.previous
.section __ex_table,"a"
.align 8
.quad 1b,30b
.quad 2b,30b
.quad 3b,30b
.quad 4b,30b
.quad 5b,30b
.quad 6b,30b
.quad 7b,30b
.quad 8b,30b
.quad 9b,30b
.quad 10b,30b
.quad 11b,30b
.quad 12b,30b
.quad 13b,30b
.quad 14b,30b
.quad 15b,30b
.quad 16b,30b
.quad 18b,40b
.quad 19b,40b
.quad 21b,50b
.quad 22b,50b
.previous
CFI_ENDPROC
ENDPROC(copy_user_generic_unrolled)
/* Some CPUs run faster using the string copy instructions.
* This is also a lot simpler. Use them when possible.
* *
* Only 4GB of copy is supported. This shouldn't be a problem * Only 4GB of copy is supported. This shouldn't be a problem
* because the kernel normally only writes from/to page sized chunks * because the kernel normally only writes from/to page sized chunks
@ -304,48 +226,43 @@ ENDPROC(copy_user_generic)
* And more would be dangerous because both Intel and AMD have * And more would be dangerous because both Intel and AMD have
* errata with rep movsq > 4GB. If someone feels the need to fix * errata with rep movsq > 4GB. If someone feels the need to fix
* this please consider this. * this please consider this.
*
* Input:
* rdi destination
* rsi source
* rdx count
*
* Output:
* eax uncopied bytes or 0 if successful.
*/ */
ENTRY(copy_user_generic_string) ENTRY(copy_user_generic_string)
CFI_STARTPROC CFI_STARTPROC
movl %ecx,%r8d /* save zero flag */ andl %edx,%edx
jz 4f
cmpl $8,%edx
jb 2f /* less than 8 bytes, go to byte copy loop */
ALIGN_DESTINATION
movl %edx,%ecx movl %edx,%ecx
shrl $3,%ecx shrl $3,%ecx
andl $7,%edx andl $7,%edx
jz 10f
1: rep 1: rep
movsq movsq
movl %edx,%ecx 2: movl %edx,%ecx
2: rep 3: rep
movsb movsb
9: movl %ecx,%eax 4: xorl %eax,%eax
ret ret
/* multiple of 8 byte */ .section .fixup,"ax"
10: rep 11: leal (%edx,%ecx,8),%ecx
movsq 12: movl %ecx,%edx /* ecx is zerorest also */
xor %eax,%eax jmp copy_user_handle_tail
ret .previous
/* exception handling */
3: lea (%rdx,%rcx,8),%rax /* exception on quad loop */
jmp 6f
5: movl %ecx,%eax /* exception on byte loop */
/* eax: left over bytes */
6: testl %r8d,%r8d /* zero flag set? */
jz 7f
movl %eax,%ecx /* initialize x86 loop counter */
push %rax
xorl %eax,%eax
8: rep
stosb /* zero the rest */
11: pop %rax
7: ret
CFI_ENDPROC
END(copy_user_generic_c)
.section __ex_table,"a" .section __ex_table,"a"
.quad 1b,3b .align 8
.quad 2b,5b .quad 1b,11b
.quad 8b,11b .quad 3b,12b
.quad 10b,3b
.previous .previous
CFI_ENDPROC
ENDPROC(copy_user_generic_string)

297
arch/x86/lib/copy_user_nocache_64.S
View File

@ -1,4 +1,6 @@
/* Copyright 2002 Andi Kleen, SuSE Labs. /*
* Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
* Copyright 2002 Andi Kleen, SuSE Labs.
* Subject to the GNU Public License v2. * Subject to the GNU Public License v2.
* *
* Functions to copy from and to user space. * Functions to copy from and to user space.
@ -12,204 +14,125 @@
#include <asm/current.h> #include <asm/current.h>
#include <asm/asm-offsets.h> #include <asm/asm-offsets.h>
#include <asm/thread_info.h> #include <asm/thread_info.h>
#include <asm/cpufeature.h>
/*
* copy_user_nocache - Uncached memory copy with exception handling
* This will force destination/source out of cache for more performance.
*
* Input:
* rdi destination
* rsi source
* rdx count
* rcx zero flag when 1 zero on exception
*
* Output:
* eax uncopied bytes or 0 if successful.
*/
ENTRY(__copy_user_nocache)
CFI_STARTPROC
pushq %rbx
CFI_ADJUST_CFA_OFFSET 8
CFI_REL_OFFSET rbx, 0
pushq %rcx /* save zero flag */
CFI_ADJUST_CFA_OFFSET 8
CFI_REL_OFFSET rcx, 0
xorl %eax,%eax /* zero for the exception handler */
.macro ALIGN_DESTINATION
#ifdef FIX_ALIGNMENT #ifdef FIX_ALIGNMENT
/* check for bad alignment of destination */ /* check for bad alignment of destination */
movl %edi,%ecx movl %edi,%ecx
andl $7,%ecx andl $7,%ecx
jnz .Lbad_alignment jz 102f /* already aligned */
.Lafter_bad_alignment: subl $8,%ecx
#endif negl %ecx
subl %ecx,%edx
movq %rdx,%rcx 100: movb (%rsi),%al
101: movb %al,(%rdi)
movl $64,%ebx
shrq $6,%rdx
decq %rdx
js .Lhandle_tail
.p2align 4
.Lloop:
.Ls1: movq (%rsi),%r11
.Ls2: movq 1*8(%rsi),%r8
.Ls3: movq 2*8(%rsi),%r9
.Ls4: movq 3*8(%rsi),%r10
.Ld1: movnti %r11,(%rdi)
.Ld2: movnti %r8,1*8(%rdi)
.Ld3: movnti %r9,2*8(%rdi)
.Ld4: movnti %r10,3*8(%rdi)
.Ls5: movq 4*8(%rsi),%r11
.Ls6: movq 5*8(%rsi),%r8
.Ls7: movq 6*8(%rsi),%r9
.Ls8: movq 7*8(%rsi),%r10
.Ld5: movnti %r11,4*8(%rdi)
.Ld6: movnti %r8,5*8(%rdi)
.Ld7: movnti %r9,6*8(%rdi)
.Ld8: movnti %r10,7*8(%rdi)
dec %rdx
leaq 64(%rsi),%rsi
leaq 64(%rdi),%rdi
jns .Lloop
.p2align 4
.Lhandle_tail:
movl %ecx,%edx
andl $63,%ecx
shrl $3,%ecx
jz .Lhandle_7
movl $8,%ebx
.p2align 4
.Lloop_8:
.Ls9: movq (%rsi),%r8
.Ld9: movnti %r8,(%rdi)
decl %ecx
leaq 8(%rdi),%rdi
leaq 8(%rsi),%rsi
jnz .Lloop_8
.Lhandle_7:
movl %edx,%ecx
andl $7,%ecx
jz .Lende
.p2align 4
.Lloop_1:
.Ls10: movb (%rsi),%bl
.Ld10: movb %bl,(%rdi)
incq %rdi
incq %rsi
decl %ecx
jnz .Lloop_1
CFI_REMEMBER_STATE
.Lende:
popq %rcx
CFI_ADJUST_CFA_OFFSET -8
CFI_RESTORE %rcx
popq %rbx
CFI_ADJUST_CFA_OFFSET -8
CFI_RESTORE rbx
sfence
ret
CFI_RESTORE_STATE
#ifdef FIX_ALIGNMENT
/* align destination */
.p2align 4
.Lbad_alignment:
movl $8,%r9d
subl %ecx,%r9d
movl %r9d,%ecx
cmpq %r9,%rdx
jz .Lhandle_7
js .Lhandle_7
.Lalign_1:
.Ls11: movb (%rsi),%bl
.Ld11: movb %bl,(%rdi)
incq %rsi incq %rsi
incq %rdi incq %rdi
decl %ecx decl %ecx
jnz .Lalign_1 jnz 100b
subq %r9,%rdx 102:
jmp .Lafter_bad_alignment .section .fixup,"ax"
#endif 103: addl %r8d,%edx /* ecx is zerorest also */
jmp copy_user_handle_tail
/* table sorted by exception address */
.section __ex_table,"a"
.align 8
.quad .Ls1,.Ls1e /* .Ls[1-4] - 0 bytes copied */
.quad .Ls2,.Ls1e
.quad .Ls3,.Ls1e
.quad .Ls4,.Ls1e
.quad .Ld1,.Ls1e /* .Ld[1-4] - 0..24 bytes coped */
.quad .Ld2,.Ls2e
.quad .Ld3,.Ls3e
.quad .Ld4,.Ls4e
.quad .Ls5,.Ls5e /* .Ls[5-8] - 32 bytes copied */
.quad .Ls6,.Ls5e
.quad .Ls7,.Ls5e
.quad .Ls8,.Ls5e
.quad .Ld5,.Ls5e /* .Ld[5-8] - 32..56 bytes copied */
.quad .Ld6,.Ls6e
.quad .Ld7,.Ls7e
.quad .Ld8,.Ls8e
.quad .Ls9,.Le_quad
.quad .Ld9,.Le_quad
.quad .Ls10,.Le_byte
.quad .Ld10,.Le_byte
#ifdef FIX_ALIGNMENT
.quad .Ls11,.Lzero_rest
.quad .Ld11,.Lzero_rest
#endif
.quad .Le5,.Le_zero
.previous .previous
/* eax: zero, ebx: 64 */ .section __ex_table,"a"
.Ls1e: addl $8,%eax /* eax: bytes left uncopied: Ls1e: 64 .. Ls8e: 8 */ .align 8
.Ls2e: addl $8,%eax .quad 100b,103b
.Ls3e: addl $8,%eax .quad 101b,103b
.Ls4e: addl $8,%eax .previous
.Ls5e: addl $8,%eax #endif
.Ls6e: addl $8,%eax .endm
.Ls7e: addl $8,%eax
.Ls8e: addl $8,%eax
addq %rbx,%rdi /* +64 */
subq %rax,%rdi /* correct destination with computed offset */
shlq $6,%rdx /* loop counter * 64 (stride length) */ /*
addq %rax,%rdx /* add offset to loopcnt */ * copy_user_nocache - Uncached memory copy with exception handling
andl $63,%ecx /* remaining bytes */ * This will force destination/source out of cache for more performance.
addq %rcx,%rdx /* add them */ */
jmp .Lzero_rest ENTRY(__copy_user_nocache)
CFI_STARTPROC
/* exception on quad word loop in tail handling */ cmpl $8,%edx
/* ecx: loopcnt/8, %edx: length, rdi: correct */ jb 20f /* less then 8 bytes, go to byte copy loop */
.Le_quad: ALIGN_DESTINATION
shll $3,%ecx movl %edx,%ecx
andl $63,%edx
shrl $6,%ecx
jz 17f
1: movq (%rsi),%r8
2: movq 1*8(%rsi),%r9
3: movq 2*8(%rsi),%r10
4: movq 3*8(%rsi),%r11
5: movnti %r8,(%rdi)
6: movnti %r9,1*8(%rdi)
7: movnti %r10,2*8(%rdi)
8: movnti %r11,3*8(%rdi)
9: movq 4*8(%rsi),%r8
10: movq 5*8(%rsi),%r9
11: movq 6*8(%rsi),%r10
12: movq 7*8(%rsi),%r11
13: movnti %r8,4*8(%rdi)
14: movnti %r9,5*8(%rdi)
15: movnti %r10,6*8(%rdi)
16: movnti %r11,7*8(%rdi)
leaq 64(%rsi),%rsi
leaq 64(%rdi),%rdi
decl %ecx
jnz 1b
17: movl %edx,%ecx
andl $7,%edx andl $7,%edx
shrl $3,%ecx
jz 20f
18: movq (%rsi),%r8
19: movnti %r8,(%rdi)
leaq 8(%rsi),%rsi
leaq 8(%rdi),%rdi
decl %ecx
jnz 18b
20: andl %edx,%edx
jz 23f
movl %edx,%ecx
21: movb (%rsi),%al
22: movb %al,(%rdi)
incq %rsi
incq %rdi
decl %ecx
jnz 21b
23: xorl %eax,%eax
sfence
ret
.section .fixup,"ax"
30: shll $6,%ecx
addl %ecx,%edx addl %ecx,%edx
/* edx: bytes to zero, rdi: dest, eax:zero */ jmp 60f
.Lzero_rest: 40: leal (%edx,%ecx,8),%edx
cmpl $0,(%rsp) /* zero flag set? */ jmp 60f
jz .Le_zero 50: movl %ecx,%edx
movq %rdx,%rcx 60: sfence
.Le_byte: movl %r8d,%ecx
xorl %eax,%eax jmp copy_user_handle_tail
.Le5: rep .previous
stosb
/* when there is another exception while zeroing the rest just return */ .section __ex_table,"a"
.Le_zero: .quad 1b,30b
movq %rdx,%rax .quad 2b,30b
jmp .Lende .quad 3b,30b
.quad 4b,30b
.quad 5b,30b
.quad 6b,30b
.quad 7b,30b
.quad 8b,30b
.quad 9b,30b
.quad 10b,30b
.quad 11b,30b
.quad 12b,30b
.quad 13b,30b
.quad 14b,30b
.quad 15b,30b
.quad 16b,30b
.quad 18b,40b
.quad 19b,40b
.quad 21b,50b
.quad 22b,50b
.previous
CFI_ENDPROC CFI_ENDPROC
ENDPROC(__copy_user_nocache) ENDPROC(__copy_user_nocache)