de5012b41e
s390 allows hotpatching the mask of a conditional jump instruction.
Make use of this feature in order to avoid the expensive stop_machine()
call.
The new trampolines are split in 3 stages:
- A first stage is a 6-byte relative conditional long branch located at
each function's entry point. Its offset always points to the second
stage for the corresponding function, and its mask is either all 0s
(ftrace off) or all 1s (ftrace on). The code for flipping the mask is
borrowed from ftrace_{enable,disable}_ftrace_graph_caller. After
flipping, ftrace_arch_code_modify_post_process() syncs with all the
other CPUs by sending SIGPs.
- Second stages for vmlinux are stored in a separate part of the .text
section reserved by the linker script, and in dynamically allocated
memory for modules. This prevents the icache pollution. The total
size of second stages is about 1.5% of that of the kernel image.
Putting second stages in the .bss section is possible and decreases
the size of the non-compressed vmlinux, but splits the kernel 1:1
mapping, which is a bad tradeoff.
Each second stage contains a call to the third stage, a pointer to
the part of the intercepted function right after the first stage, and
a pointer to an interceptor function (e.g. ftrace_caller).
Second stages are 8-byte aligned for the future direct calls
implementation.
- There are only two copies of the third stage: in the .text section
for vmlinux and in dynamically allocated memory for modules. It can be
an expoline, which is relatively large, so inlining it into each
second stage is prohibitively expensive.
As a result of this organization, phoronix-test-suite with ftrace off
does not show any performance degradation.
Suggested-by: Sven Schnelle <svens@linux.ibm.com>
Suggested-by: Vasily Gorbik <gor@linux.ibm.com>
Co-developed-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
Link: https://lore.kernel.org/r/20210728212546.128248-3-iii@linux.ibm.com
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
72 lines
1.8 KiB
C
72 lines
1.8 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_S390_FTRACE_H
|
|
#define _ASM_S390_FTRACE_H
|
|
|
|
#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
|
|
#define ARCH_SUPPORTS_FTRACE_OPS 1
|
|
#define MCOUNT_INSN_SIZE 6
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#ifdef CONFIG_CC_IS_CLANG
|
|
/* https://bugs.llvm.org/show_bug.cgi?id=41424 */
|
|
#define ftrace_return_address(n) 0UL
|
|
#else
|
|
#define ftrace_return_address(n) __builtin_return_address(n)
|
|
#endif
|
|
|
|
void ftrace_caller(void);
|
|
|
|
extern char ftrace_graph_caller_end;
|
|
extern void *ftrace_func;
|
|
|
|
struct dyn_arch_ftrace { };
|
|
|
|
#define MCOUNT_ADDR 0
|
|
#define FTRACE_ADDR ((unsigned long)ftrace_caller)
|
|
|
|
#define KPROBE_ON_FTRACE_NOP 0
|
|
#define KPROBE_ON_FTRACE_CALL 1
|
|
|
|
struct module;
|
|
struct dyn_ftrace;
|
|
|
|
bool ftrace_need_init_nop(void);
|
|
#define ftrace_need_init_nop ftrace_need_init_nop
|
|
|
|
int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec);
|
|
#define ftrace_init_nop ftrace_init_nop
|
|
|
|
static inline unsigned long ftrace_call_adjust(unsigned long addr)
|
|
{
|
|
return addr;
|
|
}
|
|
|
|
/*
|
|
* Even though the system call numbers are identical for s390/s390x a
|
|
* different system call table is used for compat tasks. This may lead
|
|
* to e.g. incorrect or missing trace event sysfs files.
|
|
* Therefore simply do not trace compat system calls at all.
|
|
* See kernel/trace/trace_syscalls.c.
|
|
*/
|
|
#define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS
|
|
static inline bool arch_trace_is_compat_syscall(struct pt_regs *regs)
|
|
{
|
|
return is_compat_task();
|
|
}
|
|
|
|
#define ARCH_HAS_SYSCALL_MATCH_SYM_NAME
|
|
static inline bool arch_syscall_match_sym_name(const char *sym,
|
|
const char *name)
|
|
{
|
|
/*
|
|
* Skip __s390_ and __s390x_ prefix - due to compat wrappers
|
|
* and aliasing some symbols of 64 bit system call functions
|
|
* may get the __s390_ prefix instead of the __s390x_ prefix.
|
|
*/
|
|
return !strcmp(sym + 7, name) || !strcmp(sym + 8, name);
|
|
}
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
#endif /* _ASM_S390_FTRACE_H */
|