mirror of
https://github.com/torvalds/linux.git
synced 2024-12-26 04:42:12 +00:00
objtool,x86: Replace alternatives with .retpoline_sites
Instead of writing complete alternatives, simply provide a list of all the retpoline thunk calls. Then the kernel is free to do with them as it pleases. Simpler code all-round. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Reviewed-by: Borislav Petkov <bp@suse.de> Acked-by: Josh Poimboeuf <jpoimboe@redhat.com> Tested-by: Alexei Starovoitov <ast@kernel.org> Link: https://lore.kernel.org/r/20211026120309.850007165@infradead.org
This commit is contained in:
Peter Zijlstra
parent
c509331b41
commit
134ab5bd18
@ -272,6 +272,20 @@ SECTIONS
|
||||
__parainstructions_end = .;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_RETPOLINE
|
||||
/*
|
||||
* List of instructions that call/jmp/jcc to retpoline thunks
|
||||
* __x86_indirect_thunk_*(). These instructions can be patched along
|
||||
* with alternatives, after which the section can be freed.
|
||||
*/
|
||||
. = ALIGN(8);
|
||||
.retpoline_sites : AT(ADDR(.retpoline_sites) - LOAD_OFFSET) {
|
||||
__retpoline_sites = .;
|
||||
*(.retpoline_sites)
|
||||
__retpoline_sites_end = .;
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* struct alt_inst entries. From the header (alternative.h):
|
||||
* "Alternative instructions for different CPU types or capabilities"
|
||||
|
||||
@ -711,126 +711,6 @@ const char *arch_ret_insn(int len)
|
||||
return ret[len-1];
|
||||
}
|
||||
|
||||
/* asm/alternative.h ? */
|
||||
|
||||
#define ALTINSTR_FLAG_INV (1 << 15)
|
||||
#define ALT_NOT(feat) ((feat) | ALTINSTR_FLAG_INV)
|
||||
|
||||
struct alt_instr {
|
||||
s32 instr_offset; /* original instruction */
|
||||
s32 repl_offset; /* offset to replacement instruction */
|
||||
u16 cpuid; /* cpuid bit set for replacement */
|
||||
u8 instrlen; /* length of original instruction */
|
||||
u8 replacementlen; /* length of new instruction */
|
||||
} __packed;
|
||||
|
||||
static int elf_add_alternative(struct elf *elf,
|
||||
struct instruction *orig, struct symbol *sym,
|
||||
int cpuid, u8 orig_len, u8 repl_len)
|
||||
{
|
||||
const int size = sizeof(struct alt_instr);
|
||||
struct alt_instr *alt;
|
||||
struct section *sec;
|
||||
Elf_Scn *s;
|
||||
|
||||
sec = find_section_by_name(elf, ".altinstructions");
|
||||
if (!sec) {
|
||||
sec = elf_create_section(elf, ".altinstructions",
|
||||
SHF_ALLOC, 0, 0);
|
||||
|
||||
if (!sec) {
|
||||
WARN_ELF("elf_create_section");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
s = elf_getscn(elf->elf, sec->idx);
|
||||
if (!s) {
|
||||
WARN_ELF("elf_getscn");
|
||||
return -1;
|
||||
}
|
||||
|
||||
sec->data = elf_newdata(s);
|
||||
if (!sec->data) {
|
||||
WARN_ELF("elf_newdata");
|
||||
return -1;
|
||||
}
|
||||
|
||||
sec->data->d_size = size;
|
||||
sec->data->d_align = 1;
|
||||
|
||||
alt = sec->data->d_buf = malloc(size);
|
||||
if (!sec->data->d_buf) {
|
||||
perror("malloc");
|
||||
return -1;
|
||||
}
|
||||
memset(sec->data->d_buf, 0, size);
|
||||
|
||||
if (elf_add_reloc_to_insn(elf, sec, sec->sh.sh_size,
|
||||
R_X86_64_PC32, orig->sec, orig->offset)) {
|
||||
WARN("elf_create_reloc: alt_instr::instr_offset");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (elf_add_reloc(elf, sec, sec->sh.sh_size + 4,
|
||||
R_X86_64_PC32, sym, 0)) {
|
||||
WARN("elf_create_reloc: alt_instr::repl_offset");
|
||||
return -1;
|
||||
}
|
||||
|
||||
alt->cpuid = bswap_if_needed(cpuid);
|
||||
alt->instrlen = orig_len;
|
||||
alt->replacementlen = repl_len;
|
||||
|
||||
sec->sh.sh_size += size;
|
||||
sec->changed = true;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define X86_FEATURE_RETPOLINE ( 7*32+12)
|
||||
|
||||
int arch_rewrite_retpolines(struct objtool_file *file)
|
||||
{
|
||||
struct instruction *insn;
|
||||
struct reloc *reloc;
|
||||
struct symbol *sym;
|
||||
char name[32] = "";
|
||||
|
||||
list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
|
||||
|
||||
if (insn->type != INSN_JUMP_DYNAMIC &&
|
||||
insn->type != INSN_CALL_DYNAMIC)
|
||||
continue;
|
||||
|
||||
if (!strcmp(insn->sec->name, ".text.__x86.indirect_thunk"))
|
||||
continue;
|
||||
|
||||
reloc = insn->reloc;
|
||||
|
||||
sprintf(name, "__x86_indirect_alt_%s_%s",
|
||||
insn->type == INSN_JUMP_DYNAMIC ? "jmp" : "call",
|
||||
reloc->sym->name + 21);
|
||||
|
||||
sym = find_symbol_by_name(file->elf, name);
|
||||
if (!sym) {
|
||||
sym = elf_create_undef_symbol(file->elf, name);
|
||||
if (!sym) {
|
||||
WARN("elf_create_undef_symbol");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (elf_add_alternative(file->elf, insn, sym,
|
||||
ALT_NOT(X86_FEATURE_RETPOLINE), 5, 5)) {
|
||||
WARN("elf_add_alternative");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int arch_decode_hint_reg(u8 sp_reg, int *base)
|
||||
{
|
||||
switch (sp_reg) {
|
||||
|
||||
@ -683,6 +683,52 @@ static int create_static_call_sections(struct objtool_file *file)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int create_retpoline_sites_sections(struct objtool_file *file)
|
||||
{
|
||||
struct instruction *insn;
|
||||
struct section *sec;
|
||||
int idx;
|
||||
|
||||
sec = find_section_by_name(file->elf, ".retpoline_sites");
|
||||
if (sec) {
|
||||
WARN("file already has .retpoline_sites, skipping");
|
||||
return 0;
|
||||
}
|
||||
|
||||
idx = 0;
|
||||
list_for_each_entry(insn, &file->retpoline_call_list, call_node)
|
||||
idx++;
|
||||
|
||||
if (!idx)
|
||||
return 0;
|
||||
|
||||
sec = elf_create_section(file->elf, ".retpoline_sites", 0,
|
||||
sizeof(int), idx);
|
||||
if (!sec) {
|
||||
WARN("elf_create_section: .retpoline_sites");
|
||||
return -1;
|
||||
}
|
||||
|
||||
idx = 0;
|
||||
list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
|
||||
|
||||
int *site = (int *)sec->data->d_buf + idx;
|
||||
*site = 0;
|
||||
|
||||
if (elf_add_reloc_to_insn(file->elf, sec,
|
||||
idx * sizeof(int),
|
||||
R_X86_64_PC32,
|
||||
insn->sec, insn->offset)) {
|
||||
WARN("elf_add_reloc_to_insn: .retpoline_sites");
|
||||
return -1;
|
||||
}
|
||||
|
||||
idx++;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int create_mcount_loc_sections(struct objtool_file *file)
|
||||
{
|
||||
struct section *sec;
|
||||
@ -1016,6 +1062,11 @@ static void annotate_call_site(struct objtool_file *file,
|
||||
return;
|
||||
}
|
||||
|
||||
if (sym->retpoline_thunk) {
|
||||
list_add_tail(&insn->call_node, &file->retpoline_call_list);
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Many compilers cannot disable KCOV with a function attribute
|
||||
* so they need a little help, NOP out any KCOV calls from noinstr
|
||||
@ -1075,6 +1126,39 @@ static void add_call_dest(struct objtool_file *file, struct instruction *insn,
|
||||
annotate_call_site(file, insn, sibling);
|
||||
}
|
||||
|
||||
static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
|
||||
{
|
||||
/*
|
||||
* Retpoline calls/jumps are really dynamic calls/jumps in disguise,
|
||||
* so convert them accordingly.
|
||||
*/
|
||||
switch (insn->type) {
|
||||
case INSN_CALL:
|
||||
insn->type = INSN_CALL_DYNAMIC;
|
||||
break;
|
||||
case INSN_JUMP_UNCONDITIONAL:
|
||||
insn->type = INSN_JUMP_DYNAMIC;
|
||||
break;
|
||||
case INSN_JUMP_CONDITIONAL:
|
||||
insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
|
||||
break;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
|
||||
insn->retpoline_safe = true;
|
||||
|
||||
/*
|
||||
* Whatever stack impact regular CALLs have, should be undone
|
||||
* by the RETURN of the called function.
|
||||
*
|
||||
* Annotated intra-function calls retain the stack_ops but
|
||||
* are converted to JUMP, see read_intra_function_calls().
|
||||
*/
|
||||
remove_insn_ops(insn);
|
||||
|
||||
annotate_call_site(file, insn, false);
|
||||
}
|
||||
/*
|
||||
* Find the destination instructions for all jumps.
|
||||
*/
|
||||
@ -1097,19 +1181,7 @@ static int add_jump_destinations(struct objtool_file *file)
|
||||
dest_sec = reloc->sym->sec;
|
||||
dest_off = arch_dest_reloc_offset(reloc->addend);
|
||||
} else if (reloc->sym->retpoline_thunk) {
|
||||
/*
|
||||
* Retpoline jumps are really dynamic jumps in
|
||||
* disguise, so convert them accordingly.
|
||||
*/
|
||||
if (insn->type == INSN_JUMP_UNCONDITIONAL)
|
||||
insn->type = INSN_JUMP_DYNAMIC;
|
||||
else
|
||||
insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
|
||||
|
||||
list_add_tail(&insn->call_node,
|
||||
&file->retpoline_call_list);
|
||||
|
||||
insn->retpoline_safe = true;
|
||||
add_retpoline_call(file, insn);
|
||||
continue;
|
||||
} else if (insn->func) {
|
||||
/* internal or external sibling call (with reloc) */
|
||||
@ -1238,18 +1310,7 @@ static int add_call_destinations(struct objtool_file *file)
|
||||
add_call_dest(file, insn, dest, false);
|
||||
|
||||
} else if (reloc->sym->retpoline_thunk) {
|
||||
/*
|
||||
* Retpoline calls are really dynamic calls in
|
||||
* disguise, so convert them accordingly.
|
||||
*/
|
||||
insn->type = INSN_CALL_DYNAMIC;
|
||||
insn->retpoline_safe = true;
|
||||
|
||||
list_add_tail(&insn->call_node,
|
||||
&file->retpoline_call_list);
|
||||
|
||||
remove_insn_ops(insn);
|
||||
continue;
|
||||
add_retpoline_call(file, insn);
|
||||
|
||||
} else
|
||||
add_call_dest(file, insn, reloc->sym, false);
|
||||
@ -1980,11 +2041,6 @@ static void mark_rodata(struct objtool_file *file)
|
||||
file->rodata = found;
|
||||
}
|
||||
|
||||
__weak int arch_rewrite_retpolines(struct objtool_file *file)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int decode_sections(struct objtool_file *file)
|
||||
{
|
||||
int ret;
|
||||
@ -2057,15 +2113,6 @@ static int decode_sections(struct objtool_file *file)
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/*
|
||||
* Must be after add_special_section_alts(), since this will emit
|
||||
* alternatives. Must be after add_{jump,call}_destination(), since
|
||||
* those create the call insn lists.
|
||||
*/
|
||||
ret = arch_rewrite_retpolines(file);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -3468,6 +3515,13 @@ int check(struct objtool_file *file)
|
||||
goto out;
|
||||
warnings += ret;
|
||||
|
||||
if (retpoline) {
|
||||
ret = create_retpoline_sites_sections(file);
|
||||
if (ret < 0)
|
||||
goto out;
|
||||
warnings += ret;
|
||||
}
|
||||
|
||||
if (mcount) {
|
||||
ret = create_mcount_loc_sections(file);
|
||||
if (ret < 0)
|
||||
|
||||
@ -740,90 +740,6 @@ static int elf_add_string(struct elf *elf, struct section *strtab, char *str)
|
||||
return len;
|
||||
}
|
||||
|
||||
struct symbol *elf_create_undef_symbol(struct elf *elf, const char *name)
|
||||
{
|
||||
struct section *symtab, *symtab_shndx;
|
||||
struct symbol *sym;
|
||||
Elf_Data *data;
|
||||
Elf_Scn *s;
|
||||
|
||||
sym = malloc(sizeof(*sym));
|
||||
if (!sym) {
|
||||
perror("malloc");
|
||||
return NULL;
|
||||
}
|
||||
memset(sym, 0, sizeof(*sym));
|
||||
|
||||
sym->name = strdup(name);
|
||||
|
||||
sym->sym.st_name = elf_add_string(elf, NULL, sym->name);
|
||||
if (sym->sym.st_name == -1)
|
||||
return NULL;
|
||||
|
||||
sym->sym.st_info = GELF_ST_INFO(STB_GLOBAL, STT_NOTYPE);
|
||||
// st_other 0
|
||||
// st_shndx 0
|
||||
// st_value 0
|
||||
// st_size 0
|
||||
|
||||
symtab = find_section_by_name(elf, ".symtab");
|
||||
if (!symtab) {
|
||||
WARN("can't find .symtab");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
s = elf_getscn(elf->elf, symtab->idx);
|
||||
if (!s) {
|
||||
WARN_ELF("elf_getscn");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
data = elf_newdata(s);
|
||||
if (!data) {
|
||||
WARN_ELF("elf_newdata");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
data->d_buf = &sym->sym;
|
||||
data->d_size = sizeof(sym->sym);
|
||||
data->d_align = 1;
|
||||
data->d_type = ELF_T_SYM;
|
||||
|
||||
sym->idx = symtab->sh.sh_size / sizeof(sym->sym);
|
||||
|
||||
symtab->sh.sh_size += data->d_size;
|
||||
symtab->changed = true;
|
||||
|
||||
symtab_shndx = find_section_by_name(elf, ".symtab_shndx");
|
||||
if (symtab_shndx) {
|
||||
s = elf_getscn(elf->elf, symtab_shndx->idx);
|
||||
if (!s) {
|
||||
WARN_ELF("elf_getscn");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
data = elf_newdata(s);
|
||||
if (!data) {
|
||||
WARN_ELF("elf_newdata");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
data->d_buf = &sym->sym.st_size; /* conveniently 0 */
|
||||
data->d_size = sizeof(Elf32_Word);
|
||||
data->d_align = 4;
|
||||
data->d_type = ELF_T_WORD;
|
||||
|
||||
symtab_shndx->sh.sh_size += 4;
|
||||
symtab_shndx->changed = true;
|
||||
}
|
||||
|
||||
sym->sec = find_section_by_index(elf, 0);
|
||||
|
||||
elf_add_symbol(elf, sym);
|
||||
|
||||
return sym;
|
||||
}
|
||||
|
||||
struct section *elf_create_section(struct elf *elf, const char *name,
|
||||
unsigned int sh_flags, size_t entsize, int nr)
|
||||
{
|
||||
|
||||
@ -144,7 +144,6 @@ int elf_write_insn(struct elf *elf, struct section *sec,
|
||||
unsigned long offset, unsigned int len,
|
||||
const char *insn);
|
||||
int elf_write_reloc(struct elf *elf, struct reloc *reloc);
|
||||
struct symbol *elf_create_undef_symbol(struct elf *elf, const char *name);
|
||||
int elf_write(struct elf *elf);
|
||||
void elf_close(struct elf *elf);
|
||||
|
||||
|
||||
@ -109,14 +109,6 @@ static int get_alt_entry(struct elf *elf, struct special_entry *entry,
|
||||
return -1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Skip retpoline .altinstr_replacement... we already rewrite the
|
||||
* instructions for retpolines anyway, see arch_is_retpoline()
|
||||
* usage in add_{call,jump}_destinations().
|
||||
*/
|
||||
if (arch_is_retpoline(new_reloc->sym))
|
||||
return 1;
|
||||
|
||||
reloc_to_sec_off(new_reloc, &alt->new_sec, &alt->new_off);
|
||||
|
||||
/* _ASM_EXTABLE_EX hack */
|
||||
|
||||
Loading…
Reference in New Issue
Block a user