f3bea49115
The current method of handling multiple function callbacks is to register a list function callback that calls all the other callbacks based on their hash tables and compare it to the function that the callback was called on. But this is very inefficient. For example, if you are tracing all functions in the kernel and then add a kprobe to a function such that the kprobe uses ftrace, the mcount trampoline will switch from calling the function trace callback to calling the list callback that will iterate over all registered ftrace_ops (in this case, the function tracer and the kprobes callback). That means for every function being traced it checks the hash of the ftrace_ops for function tracing and kprobes, even though the kprobes is only set at a single function. The kprobes ftrace_ops is checked for every function being traced! Instead of calling the list function for functions that are only being traced by a single callback, we can call a dynamically allocated trampoline that calls the callback directly. The function graph tracer already uses a direct call trampoline when it is being traced by itself but it is not dynamically allocated. It's trampoline is static in the kernel core. The infrastructure that called the function graph trampoline can also be used to call a dynamically allocated one. For now, only ftrace_ops that are not dynamically allocated can have a trampoline. That is, users such as function tracer or stack tracer. kprobes and perf allocate their ftrace_ops, and until there's a safe way to free the trampoline, it can not be used. The dynamically allocated ftrace_ops may, although, use the trampoline if the kernel is not compiled with CONFIG_PREEMPT. But that will come later. Tested-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com> Tested-by: Jiri Kosina <jkosina@suse.cz> Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
224 lines
4.1 KiB
ArmAsm
224 lines
4.1 KiB
ArmAsm
/*
|
|
* linux/arch/x86_64/mcount_64.S
|
|
*
|
|
* Copyright (C) 2014 Steven Rostedt, Red Hat Inc
|
|
*/
|
|
|
|
#include <linux/linkage.h>
|
|
#include <asm/ptrace.h>
|
|
#include <asm/ftrace.h>
|
|
|
|
|
|
.code64
|
|
.section .entry.text, "ax"
|
|
|
|
|
|
#ifdef CONFIG_FUNCTION_TRACER
|
|
|
|
#ifdef CC_USING_FENTRY
|
|
# define function_hook __fentry__
|
|
#else
|
|
# define function_hook mcount
|
|
#endif
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
|
|
|
ENTRY(function_hook)
|
|
retq
|
|
END(function_hook)
|
|
|
|
/* skip is set if stack has been adjusted */
|
|
.macro ftrace_caller_setup trace_label skip=0
|
|
MCOUNT_SAVE_FRAME \skip
|
|
|
|
/* Save this location */
|
|
GLOBAL(\trace_label)
|
|
/* Load the ftrace_ops into the 3rd parameter */
|
|
movq function_trace_op(%rip), %rdx
|
|
|
|
/* Load ip into the first parameter */
|
|
movq RIP(%rsp), %rdi
|
|
subq $MCOUNT_INSN_SIZE, %rdi
|
|
/* Load the parent_ip into the second parameter */
|
|
#ifdef CC_USING_FENTRY
|
|
movq SS+16(%rsp), %rsi
|
|
#else
|
|
movq 8(%rbp), %rsi
|
|
#endif
|
|
.endm
|
|
|
|
ENTRY(ftrace_caller)
|
|
ftrace_caller_setup ftrace_caller_op_ptr
|
|
/* regs go into 4th parameter (but make it NULL) */
|
|
movq $0, %rcx
|
|
|
|
GLOBAL(ftrace_call)
|
|
call ftrace_stub
|
|
|
|
MCOUNT_RESTORE_FRAME
|
|
|
|
/*
|
|
* The copied trampoline must call ftrace_return as it
|
|
* still may need to call the function graph tracer.
|
|
*/
|
|
GLOBAL(ftrace_caller_end)
|
|
|
|
GLOBAL(ftrace_return)
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
GLOBAL(ftrace_graph_call)
|
|
jmp ftrace_stub
|
|
#endif
|
|
|
|
GLOBAL(ftrace_stub)
|
|
retq
|
|
END(ftrace_caller)
|
|
|
|
ENTRY(ftrace_regs_caller)
|
|
/* Save the current flags before compare (in SS location)*/
|
|
pushfq
|
|
|
|
/* skip=8 to skip flags saved in SS */
|
|
ftrace_caller_setup ftrace_regs_caller_op_ptr 8
|
|
|
|
/* Save the rest of pt_regs */
|
|
movq %r15, R15(%rsp)
|
|
movq %r14, R14(%rsp)
|
|
movq %r13, R13(%rsp)
|
|
movq %r12, R12(%rsp)
|
|
movq %r11, R11(%rsp)
|
|
movq %r10, R10(%rsp)
|
|
movq %rbp, RBP(%rsp)
|
|
movq %rbx, RBX(%rsp)
|
|
/* Copy saved flags */
|
|
movq SS(%rsp), %rcx
|
|
movq %rcx, EFLAGS(%rsp)
|
|
/* Kernel segments */
|
|
movq $__KERNEL_DS, %rcx
|
|
movq %rcx, SS(%rsp)
|
|
movq $__KERNEL_CS, %rcx
|
|
movq %rcx, CS(%rsp)
|
|
/* Stack - skipping return address */
|
|
leaq SS+16(%rsp), %rcx
|
|
movq %rcx, RSP(%rsp)
|
|
|
|
/* regs go into 4th parameter */
|
|
leaq (%rsp), %rcx
|
|
|
|
GLOBAL(ftrace_regs_call)
|
|
call ftrace_stub
|
|
|
|
/* Copy flags back to SS, to restore them */
|
|
movq EFLAGS(%rsp), %rax
|
|
movq %rax, SS(%rsp)
|
|
|
|
/* Handlers can change the RIP */
|
|
movq RIP(%rsp), %rax
|
|
movq %rax, SS+8(%rsp)
|
|
|
|
/* restore the rest of pt_regs */
|
|
movq R15(%rsp), %r15
|
|
movq R14(%rsp), %r14
|
|
movq R13(%rsp), %r13
|
|
movq R12(%rsp), %r12
|
|
movq R10(%rsp), %r10
|
|
movq RBP(%rsp), %rbp
|
|
movq RBX(%rsp), %rbx
|
|
|
|
/* skip=8 to skip flags saved in SS */
|
|
MCOUNT_RESTORE_FRAME 8
|
|
|
|
/* Restore flags */
|
|
popfq
|
|
|
|
/*
|
|
* As this jmp to ftrace_return can be a short jump
|
|
* it must not be copied into the trampoline.
|
|
* The trampoline will add the code to jump
|
|
* to the return.
|
|
*/
|
|
GLOBAL(ftrace_regs_caller_end)
|
|
|
|
jmp ftrace_return
|
|
|
|
popfq
|
|
jmp ftrace_stub
|
|
|
|
END(ftrace_regs_caller)
|
|
|
|
|
|
#else /* ! CONFIG_DYNAMIC_FTRACE */
|
|
|
|
ENTRY(function_hook)
|
|
cmpq $ftrace_stub, ftrace_trace_function
|
|
jnz trace
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
cmpq $ftrace_stub, ftrace_graph_return
|
|
jnz ftrace_graph_caller
|
|
|
|
cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
|
|
jnz ftrace_graph_caller
|
|
#endif
|
|
|
|
GLOBAL(ftrace_stub)
|
|
retq
|
|
|
|
trace:
|
|
MCOUNT_SAVE_FRAME
|
|
|
|
movq RIP(%rsp), %rdi
|
|
#ifdef CC_USING_FENTRY
|
|
movq SS+16(%rsp), %rsi
|
|
#else
|
|
movq 8(%rbp), %rsi
|
|
#endif
|
|
subq $MCOUNT_INSN_SIZE, %rdi
|
|
|
|
call *ftrace_trace_function
|
|
|
|
MCOUNT_RESTORE_FRAME
|
|
|
|
jmp ftrace_stub
|
|
END(function_hook)
|
|
#endif /* CONFIG_DYNAMIC_FTRACE */
|
|
#endif /* CONFIG_FUNCTION_TRACER */
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
ENTRY(ftrace_graph_caller)
|
|
MCOUNT_SAVE_FRAME
|
|
|
|
#ifdef CC_USING_FENTRY
|
|
leaq SS+16(%rsp), %rdi
|
|
movq $0, %rdx /* No framepointers needed */
|
|
#else
|
|
leaq 8(%rbp), %rdi
|
|
movq (%rbp), %rdx
|
|
#endif
|
|
movq RIP(%rsp), %rsi
|
|
subq $MCOUNT_INSN_SIZE, %rsi
|
|
|
|
call prepare_ftrace_return
|
|
|
|
MCOUNT_RESTORE_FRAME
|
|
|
|
retq
|
|
END(ftrace_graph_caller)
|
|
|
|
GLOBAL(return_to_handler)
|
|
subq $24, %rsp
|
|
|
|
/* Save the return values */
|
|
movq %rax, (%rsp)
|
|
movq %rdx, 8(%rsp)
|
|
movq %rbp, %rdi
|
|
|
|
call ftrace_return_to_handler
|
|
|
|
movq %rax, %rdi
|
|
movq 8(%rsp), %rdx
|
|
movq (%rsp), %rax
|
|
addq $24, %rsp
|
|
jmp *%rdi
|
|
#endif
|