mirror of
https://github.com/torvalds/linux.git
synced 2024-12-27 21:33:00 +00:00
[PATCH] x86-64: Fix CFI information
Being the foundation for reliable stack unwinding, this fixes CFI unwind annotations in many low-level x86_64 routines, plus a config option (available to all architectures, and also present in the previously sent patch adding such annotations to i386 code) to enable them separatly rather than only along with adding full debug information. Signed-off-by: Jan Beulich <jbeulich@novell.com> Signed-off-by: Andi Kleen <ak@suse.de> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
This commit is contained in:
Jan Beulich
Linus Torvalds
parent
b3ab838224
commit
7effaa882a
@ -55,20 +55,34 @@
|
||||
* with the int 0x80 path.
|
||||
*/
|
||||
ENTRY(ia32_sysenter_target)
|
||||
CFI_STARTPROC
|
||||
CFI_STARTPROC simple
|
||||
CFI_DEF_CFA rsp,0
|
||||
CFI_REGISTER rsp,rbp
|
||||
swapgs
|
||||
movq %gs:pda_kernelstack, %rsp
|
||||
addq $(PDA_STACKOFFSET),%rsp
|
||||
sti
|
||||
movl %ebp,%ebp /* zero extension */
|
||||
pushq $__USER32_DS
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
/*CFI_REL_OFFSET ss,0*/
|
||||
pushq %rbp
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
CFI_REL_OFFSET rsp,0
|
||||
pushfq
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
/*CFI_REL_OFFSET rflags,0*/
|
||||
movl $VSYSCALL32_SYSEXIT, %r10d
|
||||
CFI_REGISTER rip,r10
|
||||
pushq $__USER32_CS
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
/*CFI_REL_OFFSET cs,0*/
|
||||
movl %eax, %eax
|
||||
pushq %r10
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
CFI_REL_OFFSET rip,0
|
||||
pushq %rax
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
cld
|
||||
SAVE_ARGS 0,0,1
|
||||
/* no need to do an access_ok check here because rbp has been
|
||||
@ -79,6 +93,7 @@ ENTRY(ia32_sysenter_target)
|
||||
.previous
|
||||
GET_THREAD_INFO(%r10)
|
||||
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10)
|
||||
CFI_REMEMBER_STATE
|
||||
jnz sysenter_tracesys
|
||||
sysenter_do_call:
|
||||
cmpl $(IA32_NR_syscalls),%eax
|
||||
@ -94,14 +109,20 @@ sysenter_do_call:
|
||||
andl $~0x200,EFLAGS-R11(%rsp)
|
||||
RESTORE_ARGS 1,24,1,1,1,1
|
||||
popfq
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
/*CFI_RESTORE rflags*/
|
||||
popq %rcx /* User %esp */
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
CFI_REGISTER rsp,rcx
|
||||
movl $VSYSCALL32_SYSEXIT,%edx /* User %eip */
|
||||
CFI_REGISTER rip,rdx
|
||||
swapgs
|
||||
sti /* sti only takes effect after the next instruction */
|
||||
/* sysexit */
|
||||
.byte 0xf, 0x35
|
||||
|
||||
sysenter_tracesys:
|
||||
CFI_RESTORE_STATE
|
||||
SAVE_REST
|
||||
CLEAR_RREGS
|
||||
movq $-ENOSYS,RAX(%rsp) /* really needed? */
|
||||
@ -140,21 +161,28 @@ sysenter_tracesys:
|
||||
* with the int 0x80 path.
|
||||
*/
|
||||
ENTRY(ia32_cstar_target)
|
||||
CFI_STARTPROC
|
||||
CFI_STARTPROC simple
|
||||
CFI_DEF_CFA rsp,0
|
||||
CFI_REGISTER rip,rcx
|
||||
/*CFI_REGISTER rflags,r11*/
|
||||
swapgs
|
||||
movl %esp,%r8d
|
||||
CFI_REGISTER rsp,r8
|
||||
movq %gs:pda_kernelstack,%rsp
|
||||
sti
|
||||
SAVE_ARGS 8,1,1
|
||||
movl %eax,%eax /* zero extension */
|
||||
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
|
||||
movq %rcx,RIP-ARGOFFSET(%rsp)
|
||||
CFI_REL_OFFSET rip,RIP-ARGOFFSET
|
||||
movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */
|
||||
movl %ebp,%ecx
|
||||
movq $__USER32_CS,CS-ARGOFFSET(%rsp)
|
||||
movq $__USER32_DS,SS-ARGOFFSET(%rsp)
|
||||
movq %r11,EFLAGS-ARGOFFSET(%rsp)
|
||||
/*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
|
||||
movq %r8,RSP-ARGOFFSET(%rsp)
|
||||
CFI_REL_OFFSET rsp,RSP-ARGOFFSET
|
||||
/* no need to do an access_ok check here because r8 has been
|
||||
32bit zero extended */
|
||||
/* hardware stack frame is complete now */
|
||||
@ -164,6 +192,7 @@ ENTRY(ia32_cstar_target)
|
||||
.previous
|
||||
GET_THREAD_INFO(%r10)
|
||||
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10)
|
||||
CFI_REMEMBER_STATE
|
||||
jnz cstar_tracesys
|
||||
cstar_do_call:
|
||||
cmpl $IA32_NR_syscalls,%eax
|
||||
@ -177,12 +206,16 @@ cstar_do_call:
|
||||
jnz int_ret_from_sys_call
|
||||
RESTORE_ARGS 1,-ARG_SKIP,1,1,1
|
||||
movl RIP-ARGOFFSET(%rsp),%ecx
|
||||
CFI_REGISTER rip,rcx
|
||||
movl EFLAGS-ARGOFFSET(%rsp),%r11d
|
||||
/*CFI_REGISTER rflags,r11*/
|
||||
movl RSP-ARGOFFSET(%rsp),%esp
|
||||
CFI_RESTORE rsp
|
||||
swapgs
|
||||
sysretl
|
||||
|
||||
cstar_tracesys:
|
||||
CFI_RESTORE_STATE
|
||||
SAVE_REST
|
||||
CLEAR_RREGS
|
||||
movq $-ENOSYS,RAX(%rsp) /* really needed? */
|
||||
@ -226,11 +259,18 @@ ia32_badarg:
|
||||
*/
|
||||
|
||||
ENTRY(ia32_syscall)
|
||||
CFI_STARTPROC
|
||||
CFI_STARTPROC simple
|
||||
CFI_DEF_CFA rsp,SS+8-RIP
|
||||
/*CFI_REL_OFFSET ss,SS-RIP*/
|
||||
CFI_REL_OFFSET rsp,RSP-RIP
|
||||
/*CFI_REL_OFFSET rflags,EFLAGS-RIP*/
|
||||
/*CFI_REL_OFFSET cs,CS-RIP*/
|
||||
CFI_REL_OFFSET rip,RIP-RIP
|
||||
swapgs
|
||||
sti
|
||||
movl %eax,%eax
|
||||
pushq %rax
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
cld
|
||||
/* note the registers are not zero extended to the sf.
|
||||
this could be a problem. */
|
||||
@ -278,6 +318,8 @@ quiet_ni_syscall:
|
||||
jmp ia32_ptregs_common
|
||||
.endm
|
||||
|
||||
CFI_STARTPROC
|
||||
|
||||
PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi
|
||||
PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi
|
||||
PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx
|
||||
@ -290,8 +332,9 @@ quiet_ni_syscall:
|
||||
PTREGSCALL stub32_rt_sigsuspend, sys_rt_sigsuspend, %rdx
|
||||
|
||||
ENTRY(ia32_ptregs_common)
|
||||
CFI_STARTPROC
|
||||
popq %r11
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
CFI_REGISTER rip, r11
|
||||
SAVE_REST
|
||||
call *%rax
|
||||
RESTORE_REST
|
||||
|
||||
@ -79,16 +79,19 @@
|
||||
xorl %eax, %eax
|
||||
pushq %rax /* ss */
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
/*CFI_REL_OFFSET ss,0*/
|
||||
pushq %rax /* rsp */
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
CFI_OFFSET rip,0
|
||||
CFI_REL_OFFSET rsp,0
|
||||
pushq $(1<<9) /* eflags - interrupts on */
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
/*CFI_REL_OFFSET rflags,0*/
|
||||
pushq $__KERNEL_CS /* cs */
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
/*CFI_REL_OFFSET cs,0*/
|
||||
pushq \child_rip /* rip */
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
CFI_OFFSET rip,0
|
||||
CFI_REL_OFFSET rip,0
|
||||
pushq %rax /* orig rax */
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
.endm
|
||||
@ -98,32 +101,39 @@
|
||||
CFI_ADJUST_CFA_OFFSET -(6*8)
|
||||
.endm
|
||||
|
||||
.macro CFI_DEFAULT_STACK
|
||||
CFI_ADJUST_CFA_OFFSET (SS)
|
||||
CFI_OFFSET r15,R15-SS
|
||||
CFI_OFFSET r14,R14-SS
|
||||
CFI_OFFSET r13,R13-SS
|
||||
CFI_OFFSET r12,R12-SS
|
||||
CFI_OFFSET rbp,RBP-SS
|
||||
CFI_OFFSET rbx,RBX-SS
|
||||
CFI_OFFSET r11,R11-SS
|
||||
CFI_OFFSET r10,R10-SS
|
||||
CFI_OFFSET r9,R9-SS
|
||||
CFI_OFFSET r8,R8-SS
|
||||
CFI_OFFSET rax,RAX-SS
|
||||
CFI_OFFSET rcx,RCX-SS
|
||||
CFI_OFFSET rdx,RDX-SS
|
||||
CFI_OFFSET rsi,RSI-SS
|
||||
CFI_OFFSET rdi,RDI-SS
|
||||
CFI_OFFSET rsp,RSP-SS
|
||||
CFI_OFFSET rip,RIP-SS
|
||||
.macro CFI_DEFAULT_STACK start=1
|
||||
.if \start
|
||||
CFI_STARTPROC simple
|
||||
CFI_DEF_CFA rsp,SS+8
|
||||
.else
|
||||
CFI_DEF_CFA_OFFSET SS+8
|
||||
.endif
|
||||
CFI_REL_OFFSET r15,R15
|
||||
CFI_REL_OFFSET r14,R14
|
||||
CFI_REL_OFFSET r13,R13
|
||||
CFI_REL_OFFSET r12,R12
|
||||
CFI_REL_OFFSET rbp,RBP
|
||||
CFI_REL_OFFSET rbx,RBX
|
||||
CFI_REL_OFFSET r11,R11
|
||||
CFI_REL_OFFSET r10,R10
|
||||
CFI_REL_OFFSET r9,R9
|
||||
CFI_REL_OFFSET r8,R8
|
||||
CFI_REL_OFFSET rax,RAX
|
||||
CFI_REL_OFFSET rcx,RCX
|
||||
CFI_REL_OFFSET rdx,RDX
|
||||
CFI_REL_OFFSET rsi,RSI
|
||||
CFI_REL_OFFSET rdi,RDI
|
||||
CFI_REL_OFFSET rip,RIP
|
||||
/*CFI_REL_OFFSET cs,CS*/
|
||||
/*CFI_REL_OFFSET rflags,EFLAGS*/
|
||||
CFI_REL_OFFSET rsp,RSP
|
||||
/*CFI_REL_OFFSET ss,SS*/
|
||||
.endm
|
||||
/*
|
||||
* A newly forked process directly context switches into this.
|
||||
*/
|
||||
/* rdi: prev */
|
||||
ENTRY(ret_from_fork)
|
||||
CFI_STARTPROC
|
||||
CFI_DEFAULT_STACK
|
||||
call schedule_tail
|
||||
GET_THREAD_INFO(%rcx)
|
||||
@ -172,16 +182,21 @@ rff_trace:
|
||||
*/
|
||||
|
||||
ENTRY(system_call)
|
||||
CFI_STARTPROC
|
||||
CFI_STARTPROC simple
|
||||
CFI_DEF_CFA rsp,0
|
||||
CFI_REGISTER rip,rcx
|
||||
/*CFI_REGISTER rflags,r11*/
|
||||
swapgs
|
||||
movq %rsp,%gs:pda_oldrsp
|
||||
movq %gs:pda_kernelstack,%rsp
|
||||
sti
|
||||
SAVE_ARGS 8,1
|
||||
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
|
||||
movq %rcx,RIP-ARGOFFSET(%rsp)
|
||||
movq %rcx,RIP-ARGOFFSET(%rsp)
|
||||
CFI_REL_OFFSET rip,RIP-ARGOFFSET
|
||||
GET_THREAD_INFO(%rcx)
|
||||
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
|
||||
CFI_REMEMBER_STATE
|
||||
jnz tracesys
|
||||
cmpq $__NR_syscall_max,%rax
|
||||
ja badsys
|
||||
@ -201,9 +216,12 @@ sysret_check:
|
||||
cli
|
||||
movl threadinfo_flags(%rcx),%edx
|
||||
andl %edi,%edx
|
||||
CFI_REMEMBER_STATE
|
||||
jnz sysret_careful
|
||||
movq RIP-ARGOFFSET(%rsp),%rcx
|
||||
CFI_REGISTER rip,rcx
|
||||
RESTORE_ARGS 0,-ARG_SKIP,1
|
||||
/*CFI_REGISTER rflags,r11*/
|
||||
movq %gs:pda_oldrsp,%rsp
|
||||
swapgs
|
||||
sysretq
|
||||
@ -211,12 +229,15 @@ sysret_check:
|
||||
/* Handle reschedules */
|
||||
/* edx: work, edi: workmask */
|
||||
sysret_careful:
|
||||
CFI_RESTORE_STATE
|
||||
bt $TIF_NEED_RESCHED,%edx
|
||||
jnc sysret_signal
|
||||
sti
|
||||
pushq %rdi
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
call schedule
|
||||
popq %rdi
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
jmp sysret_check
|
||||
|
||||
/* Handle a signal */
|
||||
@ -234,8 +255,13 @@ sysret_signal:
|
||||
1: movl $_TIF_NEED_RESCHED,%edi
|
||||
jmp sysret_check
|
||||
|
||||
badsys:
|
||||
movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
|
||||
jmp ret_from_sys_call
|
||||
|
||||
/* Do syscall tracing */
|
||||
tracesys:
|
||||
CFI_RESTORE_STATE
|
||||
SAVE_REST
|
||||
movq $-ENOSYS,RAX(%rsp)
|
||||
FIXUP_TOP_OF_STACK %rdi
|
||||
@ -254,16 +280,29 @@ tracesys:
|
||||
RESTORE_TOP_OF_STACK %rbx
|
||||
RESTORE_REST
|
||||
jmp ret_from_sys_call
|
||||
CFI_ENDPROC
|
||||
|
||||
badsys:
|
||||
movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
|
||||
jmp ret_from_sys_call
|
||||
|
||||
/*
|
||||
* Syscall return path ending with IRET.
|
||||
* Has correct top of stack, but partial stack frame.
|
||||
*/
|
||||
ENTRY(int_ret_from_sys_call)
|
||||
ENTRY(int_ret_from_sys_call)
|
||||
CFI_STARTPROC simple
|
||||
CFI_DEF_CFA rsp,SS+8-ARGOFFSET
|
||||
/*CFI_REL_OFFSET ss,SS-ARGOFFSET*/
|
||||
CFI_REL_OFFSET rsp,RSP-ARGOFFSET
|
||||
/*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
|
||||
/*CFI_REL_OFFSET cs,CS-ARGOFFSET*/
|
||||
CFI_REL_OFFSET rip,RIP-ARGOFFSET
|
||||
CFI_REL_OFFSET rdx,RDX-ARGOFFSET
|
||||
CFI_REL_OFFSET rcx,RCX-ARGOFFSET
|
||||
CFI_REL_OFFSET rax,RAX-ARGOFFSET
|
||||
CFI_REL_OFFSET rdi,RDI-ARGOFFSET
|
||||
CFI_REL_OFFSET rsi,RSI-ARGOFFSET
|
||||
CFI_REL_OFFSET r8,R8-ARGOFFSET
|
||||
CFI_REL_OFFSET r9,R9-ARGOFFSET
|
||||
CFI_REL_OFFSET r10,R10-ARGOFFSET
|
||||
CFI_REL_OFFSET r11,R11-ARGOFFSET
|
||||
cli
|
||||
testl $3,CS-ARGOFFSET(%rsp)
|
||||
je retint_restore_args
|
||||
@ -284,8 +323,10 @@ int_careful:
|
||||
jnc int_very_careful
|
||||
sti
|
||||
pushq %rdi
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
call schedule
|
||||
popq %rdi
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
cli
|
||||
jmp int_with_check
|
||||
|
||||
@ -297,9 +338,11 @@ int_very_careful:
|
||||
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
|
||||
jz int_signal
|
||||
pushq %rdi
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
leaq 8(%rsp),%rdi # &ptregs -> arg1
|
||||
call syscall_trace_leave
|
||||
popq %rdi
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
|
||||
cli
|
||||
jmp int_restore_rest
|
||||
@ -329,6 +372,8 @@ int_restore_rest:
|
||||
jmp ptregscall_common
|
||||
.endm
|
||||
|
||||
CFI_STARTPROC
|
||||
|
||||
PTREGSCALL stub_clone, sys_clone, %r8
|
||||
PTREGSCALL stub_fork, sys_fork, %rdi
|
||||
PTREGSCALL stub_vfork, sys_vfork, %rdi
|
||||
@ -337,40 +382,49 @@ int_restore_rest:
|
||||
PTREGSCALL stub_iopl, sys_iopl, %rsi
|
||||
|
||||
ENTRY(ptregscall_common)
|
||||
CFI_STARTPROC
|
||||
popq %r11
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
CFI_REGISTER rip, r11
|
||||
SAVE_REST
|
||||
movq %r11, %r15
|
||||
CFI_REGISTER rip, r15
|
||||
FIXUP_TOP_OF_STACK %r11
|
||||
call *%rax
|
||||
RESTORE_TOP_OF_STACK %r11
|
||||
movq %r15, %r11
|
||||
CFI_REGISTER rip, r11
|
||||
RESTORE_REST
|
||||
pushq %r11
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
CFI_REL_OFFSET rip, 0
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
|
||||
ENTRY(stub_execve)
|
||||
CFI_STARTPROC
|
||||
popq %r11
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
CFI_REGISTER rip, r11
|
||||
SAVE_REST
|
||||
movq %r11, %r15
|
||||
CFI_REGISTER rip, r15
|
||||
FIXUP_TOP_OF_STACK %r11
|
||||
call sys_execve
|
||||
GET_THREAD_INFO(%rcx)
|
||||
bt $TIF_IA32,threadinfo_flags(%rcx)
|
||||
CFI_REMEMBER_STATE
|
||||
jc exec_32bit
|
||||
RESTORE_TOP_OF_STACK %r11
|
||||
movq %r15, %r11
|
||||
CFI_REGISTER rip, r11
|
||||
RESTORE_REST
|
||||
push %r11
|
||||
pushq %r11
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
CFI_REL_OFFSET rip, 0
|
||||
ret
|
||||
|
||||
exec_32bit:
|
||||
CFI_ADJUST_CFA_OFFSET REST_SKIP
|
||||
CFI_RESTORE_STATE
|
||||
movq %rax,RAX(%rsp)
|
||||
RESTORE_REST
|
||||
jmp int_ret_from_sys_call
|
||||
@ -382,7 +436,8 @@ exec_32bit:
|
||||
*/
|
||||
ENTRY(stub_rt_sigreturn)
|
||||
CFI_STARTPROC
|
||||
addq $8, %rsp
|
||||
addq $8, %rsp
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
SAVE_REST
|
||||
movq %rsp,%rdi
|
||||
FIXUP_TOP_OF_STACK %r11
|
||||
@ -392,6 +447,25 @@ ENTRY(stub_rt_sigreturn)
|
||||
jmp int_ret_from_sys_call
|
||||
CFI_ENDPROC
|
||||
|
||||
/*
|
||||
* initial frame state for interrupts and exceptions
|
||||
*/
|
||||
.macro _frame ref
|
||||
CFI_STARTPROC simple
|
||||
CFI_DEF_CFA rsp,SS+8-\ref
|
||||
/*CFI_REL_OFFSET ss,SS-\ref*/
|
||||
CFI_REL_OFFSET rsp,RSP-\ref
|
||||
/*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
|
||||
/*CFI_REL_OFFSET cs,CS-\ref*/
|
||||
CFI_REL_OFFSET rip,RIP-\ref
|
||||
.endm
|
||||
|
||||
/* initial frame state for interrupts (and exceptions without error code) */
|
||||
#define INTR_FRAME _frame RIP
|
||||
/* initial frame state for exceptions with error code (and interrupts with
|
||||
vector already pushed) */
|
||||
#define XCPT_FRAME _frame ORIG_RAX
|
||||
|
||||
/*
|
||||
* Interrupt entry/exit.
|
||||
*
|
||||
@ -402,10 +476,6 @@ ENTRY(stub_rt_sigreturn)
|
||||
|
||||
/* 0(%rsp): interrupt number */
|
||||
.macro interrupt func
|
||||
CFI_STARTPROC simple
|
||||
CFI_DEF_CFA rsp,(SS-RDI)
|
||||
CFI_REL_OFFSET rsp,(RSP-ORIG_RAX)
|
||||
CFI_REL_OFFSET rip,(RIP-ORIG_RAX)
|
||||
cld
|
||||
#ifdef CONFIG_DEBUG_INFO
|
||||
SAVE_ALL
|
||||
@ -425,23 +495,27 @@ ENTRY(stub_rt_sigreturn)
|
||||
swapgs
|
||||
1: incl %gs:pda_irqcount # RED-PEN should check preempt count
|
||||
movq %gs:pda_irqstackptr,%rax
|
||||
cmoveq %rax,%rsp
|
||||
cmoveq %rax,%rsp /*todo This needs CFI annotation! */
|
||||
pushq %rdi # save old stack
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
call \func
|
||||
.endm
|
||||
|
||||
ENTRY(common_interrupt)
|
||||
XCPT_FRAME
|
||||
interrupt do_IRQ
|
||||
/* 0(%rsp): oldrsp-ARGOFFSET */
|
||||
ret_from_intr:
|
||||
ret_from_intr:
|
||||
popq %rdi
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
cli
|
||||
decl %gs:pda_irqcount
|
||||
#ifdef CONFIG_DEBUG_INFO
|
||||
movq RBP(%rdi),%rbp
|
||||
CFI_DEF_CFA_REGISTER rsp
|
||||
#endif
|
||||
leaq ARGOFFSET(%rdi),%rsp
|
||||
exit_intr:
|
||||
leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
|
||||
exit_intr:
|
||||
GET_THREAD_INFO(%rcx)
|
||||
testl $3,CS-ARGOFFSET(%rsp)
|
||||
je retint_kernel
|
||||
@ -453,9 +527,10 @@ exit_intr:
|
||||
*/
|
||||
retint_with_reschedule:
|
||||
movl $_TIF_WORK_MASK,%edi
|
||||
retint_check:
|
||||
retint_check:
|
||||
movl threadinfo_flags(%rcx),%edx
|
||||
andl %edi,%edx
|
||||
CFI_REMEMBER_STATE
|
||||
jnz retint_careful
|
||||
retint_swapgs:
|
||||
swapgs
|
||||
@ -476,14 +551,17 @@ bad_iret:
|
||||
jmp do_exit
|
||||
.previous
|
||||
|
||||
/* edi: workmask, edx: work */
|
||||
/* edi: workmask, edx: work */
|
||||
retint_careful:
|
||||
CFI_RESTORE_STATE
|
||||
bt $TIF_NEED_RESCHED,%edx
|
||||
jnc retint_signal
|
||||
sti
|
||||
pushq %rdi
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
call schedule
|
||||
popq %rdi
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
GET_THREAD_INFO(%rcx)
|
||||
cli
|
||||
jmp retint_check
|
||||
@ -523,7 +601,9 @@ retint_kernel:
|
||||
* APIC interrupts.
|
||||
*/
|
||||
.macro apicinterrupt num,func
|
||||
INTR_FRAME
|
||||
pushq $\num-256
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
interrupt \func
|
||||
jmp ret_from_intr
|
||||
CFI_ENDPROC
|
||||
@ -569,16 +649,23 @@ ENTRY(spurious_interrupt)
|
||||
* Exception entry points.
|
||||
*/
|
||||
.macro zeroentry sym
|
||||
INTR_FRAME
|
||||
pushq $0 /* push error code/oldrax */
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
pushq %rax /* push real oldrax to the rdi slot */
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
leaq \sym(%rip),%rax
|
||||
jmp error_entry
|
||||
CFI_ENDPROC
|
||||
.endm
|
||||
|
||||
.macro errorentry sym
|
||||
XCPT_FRAME
|
||||
pushq %rax
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
leaq \sym(%rip),%rax
|
||||
jmp error_entry
|
||||
CFI_ENDPROC
|
||||
.endm
|
||||
|
||||
/* error code is on the stack already */
|
||||
@ -605,10 +692,7 @@ ENTRY(spurious_interrupt)
|
||||
* and the exception handler in %rax.
|
||||
*/
|
||||
ENTRY(error_entry)
|
||||
CFI_STARTPROC simple
|
||||
CFI_DEF_CFA rsp,(SS-RDI)
|
||||
CFI_REL_OFFSET rsp,(RSP-RDI)
|
||||
CFI_REL_OFFSET rip,(RIP-RDI)
|
||||
_frame RDI
|
||||
/* rdi slot contains rax, oldrax contains error code */
|
||||
cld
|
||||
subq $14*8,%rsp
|
||||
@ -690,7 +774,9 @@ error_kernelspace:
|
||||
/* Reload gs selector with exception handling */
|
||||
/* edi: new selector */
|
||||
ENTRY(load_gs_index)
|
||||
CFI_STARTPROC
|
||||
pushf
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
cli
|
||||
swapgs
|
||||
gs_change:
|
||||
@ -698,7 +784,9 @@ gs_change:
|
||||
2: mfence /* workaround */
|
||||
swapgs
|
||||
popf
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
|
||||
.section __ex_table,"a"
|
||||
.align 8
|
||||
@ -810,7 +898,7 @@ ENTRY(device_not_available)
|
||||
|
||||
/* runs on exception stack */
|
||||
KPROBE_ENTRY(debug)
|
||||
CFI_STARTPROC
|
||||
INTR_FRAME
|
||||
pushq $0
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
paranoidentry do_debug
|
||||
@ -820,9 +908,9 @@ KPROBE_ENTRY(debug)
|
||||
|
||||
/* runs on exception stack */
|
||||
ENTRY(nmi)
|
||||
CFI_STARTPROC
|
||||
INTR_FRAME
|
||||
pushq $-1
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
paranoidentry do_nmi
|
||||
/*
|
||||
* "Paranoid" exit path from exception stack.
|
||||
@ -888,7 +976,7 @@ ENTRY(reserved)
|
||||
|
||||
/* runs on exception stack */
|
||||
ENTRY(double_fault)
|
||||
CFI_STARTPROC
|
||||
XCPT_FRAME
|
||||
paranoidentry do_double_fault
|
||||
jmp paranoid_exit
|
||||
CFI_ENDPROC
|
||||
@ -901,7 +989,7 @@ ENTRY(segment_not_present)
|
||||
|
||||
/* runs on exception stack */
|
||||
ENTRY(stack_segment)
|
||||
CFI_STARTPROC
|
||||
XCPT_FRAME
|
||||
paranoidentry do_stack_segment
|
||||
jmp paranoid_exit
|
||||
CFI_ENDPROC
|
||||
@ -922,7 +1010,7 @@ ENTRY(spurious_interrupt_bug)
|
||||
#ifdef CONFIG_X86_MCE
|
||||
/* runs on exception stack */
|
||||
ENTRY(machine_check)
|
||||
CFI_STARTPROC
|
||||
INTR_FRAME
|
||||
pushq $0
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
paranoidentry do_machine_check
|
||||
@ -934,14 +1022,19 @@ ENTRY(call_debug)
|
||||
zeroentry do_call_debug
|
||||
|
||||
ENTRY(call_softirq)
|
||||
CFI_STARTPROC
|
||||
movq %gs:pda_irqstackptr,%rax
|
||||
pushq %r15
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
movq %rsp,%r15
|
||||
CFI_DEF_CFA_REGISTER r15
|
||||
incl %gs:pda_irqcount
|
||||
cmove %rax,%rsp
|
||||
call __do_softirq
|
||||
movq %r15,%rsp
|
||||
CFI_DEF_CFA_REGISTER rsp
|
||||
decl %gs:pda_irqcount
|
||||
popq %r15
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
ret
|
||||
|
||||
CFI_ENDPROC
|
||||
|
||||
@ -65,27 +65,36 @@
|
||||
.if \skipr11
|
||||
.else
|
||||
movq (%rsp),%r11
|
||||
CFI_RESTORE r11
|
||||
.endif
|
||||
.if \skipr8910
|
||||
.else
|
||||
movq 1*8(%rsp),%r10
|
||||
CFI_RESTORE r10
|
||||
movq 2*8(%rsp),%r9
|
||||
CFI_RESTORE r9
|
||||
movq 3*8(%rsp),%r8
|
||||
CFI_RESTORE r8
|
||||
.endif
|
||||
.if \skiprax
|
||||
.else
|
||||
movq 4*8(%rsp),%rax
|
||||
CFI_RESTORE rax
|
||||
.endif
|
||||
.if \skiprcx
|
||||
.else
|
||||
movq 5*8(%rsp),%rcx
|
||||
CFI_RESTORE rcx
|
||||
.endif
|
||||
.if \skiprdx
|
||||
.else
|
||||
movq 6*8(%rsp),%rdx
|
||||
CFI_RESTORE rdx
|
||||
.endif
|
||||
movq 7*8(%rsp),%rsi
|
||||
CFI_RESTORE rsi
|
||||
movq 8*8(%rsp),%rdi
|
||||
CFI_RESTORE rdi
|
||||
.if ARG_SKIP+\addskip > 0
|
||||
addq $ARG_SKIP+\addskip,%rsp
|
||||
CFI_ADJUST_CFA_OFFSET -(ARG_SKIP+\addskip)
|
||||
@ -124,11 +133,17 @@
|
||||
|
||||
.macro RESTORE_REST
|
||||
movq (%rsp),%r15
|
||||
CFI_RESTORE r15
|
||||
movq 1*8(%rsp),%r14
|
||||
CFI_RESTORE r14
|
||||
movq 2*8(%rsp),%r13
|
||||
CFI_RESTORE r13
|
||||
movq 3*8(%rsp),%r12
|
||||
CFI_RESTORE r12
|
||||
movq 4*8(%rsp),%rbp
|
||||
CFI_RESTORE rbp
|
||||
movq 5*8(%rsp),%rbx
|
||||
CFI_RESTORE rbx
|
||||
addq $REST_SKIP,%rsp
|
||||
CFI_ADJUST_CFA_OFFSET -(REST_SKIP)
|
||||
.endm
|
||||
@ -146,11 +161,3 @@
|
||||
.macro icebp
|
||||
.byte 0xf1
|
||||
.endm
|
||||
|
||||
#ifdef CONFIG_FRAME_POINTER
|
||||
#define ENTER enter
|
||||
#define LEAVE leave
|
||||
#else
|
||||
#define ENTER
|
||||
#define LEAVE
|
||||
#endif
|
||||
|
||||
@ -24,6 +24,10 @@
|
||||
#define CFI_ADJUST_CFA_OFFSET .cfi_adjust_cfa_offset
|
||||
#define CFI_OFFSET .cfi_offset
|
||||
#define CFI_REL_OFFSET .cfi_rel_offset
|
||||
#define CFI_REGISTER .cfi_register
|
||||
#define CFI_RESTORE .cfi_restore
|
||||
#define CFI_REMEMBER_STATE .cfi_remember_state
|
||||
#define CFI_RESTORE_STATE .cfi_restore_state
|
||||
|
||||
#else
|
||||
|
||||
@ -36,6 +40,10 @@
|
||||
#define CFI_ADJUST_CFA_OFFSET #
|
||||
#define CFI_OFFSET #
|
||||
#define CFI_REL_OFFSET #
|
||||
#define CFI_REGISTER #
|
||||
#define CFI_RESTORE #
|
||||
#define CFI_REMEMBER_STATE #
|
||||
#define CFI_RESTORE_STATE #
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user