Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-27 13:22:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'for-next/bti' into for-next/core

Browse Source 
Support for Branch Target Identification (BTI) in user and kernel
(Mark Brown and others)
* for-next/bti: (39 commits)
  arm64: vdso: Fix CFI directives in sigreturn trampoline
  arm64: vdso: Don't prefix sigreturn trampoline with a BTI C instruction
  arm64: bti: Fix support for userspace only BTI
  arm64: kconfig: Update and comment GCC version check for kernel BTI
  arm64: vdso: Map the vDSO text with guarded pages when built for BTI
  arm64: vdso: Force the vDSO to be linked as BTI when built for BTI
  arm64: vdso: Annotate for BTI
  arm64: asm: Provide a mechanism for generating ELF note for BTI
  arm64: bti: Provide Kconfig for kernel mode BTI
  arm64: mm: Mark executable text as guarded pages
  arm64: bpf: Annotate JITed code for BTI
  arm64: Set GP bit in kernel page tables to enable BTI for the kernel
  arm64: asm: Override SYM_FUNC_START when building the kernel with BTI
  arm64: bti: Support building kernel C code using BTI
  arm64: Document why we enable PAC support for leaf functions
  arm64: insn: Report PAC and BTI instructions as skippable
  arm64: insn: Don't assume unrecognized HINTs are skippable
  arm64: insn: Provide a better name for aarch64_insn_is_nop()
  arm64: insn: Add constants for new HINT instruction decode
  arm64: Disable old style assembly annotations
  ...
This commit is contained in:
Will Deacon 2020-05-28 18:00:51 +01:00
commit d27865279f
75 changed files with 963 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Documentation/arm64/cpu-feature-registers.rst
View File

@ -176,6 +176,8 @@ infrastructure:
+------------------------------+---------+---------+
| SSBS | [7-4] | y |
+------------------------------+---------+---------+
| BT | [3-0] | y |
+------------------------------+---------+---------+
4) MIDR_EL1 - Main ID Register

5
Documentation/arm64/elf_hwcaps.rst
View File

@ -236,6 +236,11 @@ HWCAP2_RNG
Functionality implied by ID_AA64ISAR0_EL1.RNDR == 0b0001.
HWCAP2_BTI
Functionality implied by ID_AA64PFR0_EL1.BT == 0b0001.
4. Unused AT_HWCAP bits
-----------------------

1
Documentation/filesystems/proc.rst
View File

@ -543,6 +543,7 @@ encoded manner. The codes are the following:
hg huge page advise flag
nh no huge page advise flag
mg mergable advise flag
bt - arm64 BTI guarded page
== =======================================
Note that there is no guarantee that every flag and associated mnemonic will

46
arch/arm64/Kconfig
View File

@ -9,6 +9,7 @@ config ARM64
select ACPI_MCFG if (ACPI && PCI)
select ACPI_SPCR_TABLE if ACPI
select ACPI_PPTT if ACPI
select ARCH_BINFMT_ELF_STATE
select ARCH_HAS_DEBUG_VIRTUAL
select ARCH_HAS_DEVMEM_IS_ALLOWED
select ARCH_HAS_DMA_PREP_COHERENT
@ -32,6 +33,7 @@ config ARM64
select ARCH_HAS_SYSCALL_WRAPPER
select ARCH_HAS_TEARDOWN_DMA_OPS if IOMMU_SUPPORT
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
select ARCH_HAVE_ELF_PROT
select ARCH_HAVE_NMI_SAFE_CMPXCHG
select ARCH_INLINE_READ_LOCK if !PREEMPTION
select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION
@ -61,8 +63,10 @@ config ARM64
select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION
select ARCH_KEEP_MEMBLOCK
select ARCH_USE_CMPXCHG_LOCKREF
select ARCH_USE_GNU_PROPERTY
select ARCH_USE_QUEUED_RWLOCKS
select ARCH_USE_QUEUED_SPINLOCKS
select ARCH_USE_SYM_ANNOTATIONS
select ARCH_SUPPORTS_MEMORY_FAILURE
select ARCH_SUPPORTS_ATOMIC_RMW
select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
@ -1584,6 +1588,48 @@ endmenu
menu "ARMv8.5 architectural features"
config ARM64_BTI
bool "Branch Target Identification support"
default y
help
Branch Target Identification (part of the ARMv8.5 Extensions)
provides a mechanism to limit the set of locations to which computed
branch instructions such as BR or BLR can jump.
To make use of BTI on CPUs that support it, say Y.
BTI is intended to provide complementary protection to other control
flow integrity protection mechanisms, such as the Pointer
authentication mechanism provided as part of the ARMv8.3 Extensions.
For this reason, it does not make sense to enable this option without
also enabling support for pointer authentication. Thus, when
enabling this option you should also select ARM64_PTR_AUTH=y.
Userspace binaries must also be specifically compiled to make use of
this mechanism. If you say N here or the hardware does not support
BTI, such binaries can still run, but you get no additional
enforcement of branch destinations.
config ARM64_BTI_KERNEL
bool "Use Branch Target Identification for kernel"
default y
depends on ARM64_BTI
depends on ARM64_PTR_AUTH
depends on CC_HAS_BRANCH_PROT_PAC_RET_BTI
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94697
depends on !CC_IS_GCC || GCC_VERSION >= 100100
depends on !(CC_IS_CLANG && GCOV_KERNEL)
depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS)
help
Build the kernel with Branch Target Identification annotations
and enable enforcement of this for kernel code. When this option
is enabled and the system supports BTI all kernel code including
modular code must have BTI enabled.
config CC_HAS_BRANCH_PROT_PAC_RET_BTI
# GCC 9 or later, clang 8 or later
def_bool $(cc-option,-mbranch-protection=pac-ret+leaf+bti)
config ARM64_E0PD
bool "Enable support for E0PD"
default y

7
arch/arm64/Makefile
View File

@ -70,7 +70,14 @@ branch-prot-flags-y += $(call cc-option,-mbranch-protection=none)
ifeq ($(CONFIG_ARM64_PTR_AUTH),y)
branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address=all
# We enable additional protection for leaf functions as there is some
# narrow potential for ROP protection benefits and no substantial
# performance impact has been observed.
ifeq ($(CONFIG_ARM64_BTI_KERNEL),y)
branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=pac-ret+leaf+bti
else
branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=pac-ret+leaf
endif
# -march=armv8.3-a enables the non-nops instructions for PAC, to avoid the
# compiler to generate them and consequently to break the single image contract
# we pass it only to the assembler. This option is utilized only in case of non

50
arch/arm64/include/asm/assembler.h
View File

@ -736,4 +736,54 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU
.Lyield_out_\@ :
.endm
/*
* This macro emits a program property note section identifying
* architecture features which require special handling, mainly for
* use in assembly files included in the VDSO.
*/
#define NT_GNU_PROPERTY_TYPE_0 5
#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0)
#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1)
#ifdef CONFIG_ARM64_BTI_KERNEL
#define GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT \
((GNU_PROPERTY_AARCH64_FEATURE_1_BTI | \
GNU_PROPERTY_AARCH64_FEATURE_1_PAC))
#endif
#ifdef GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT
.macro emit_aarch64_feature_1_and, feat=GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT
.pushsection .note.gnu.property, "a"
.align 3
.long 2f - 1f
.long 6f - 3f
.long NT_GNU_PROPERTY_TYPE_0
1: .string "GNU"
2:
.align 3
3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND
.long 5f - 4f
4:
/*
* This is described with an array of char in the Linux API
* spec but the text and all other usage (including binutils,
* clang and GCC) treat this as a 32 bit value so no swizzling
* is required for big endian.
*/
.long \feat
5:
.align 3
6:
.popsection
.endm
#else
.macro emit_aarch64_feature_1_and, feat=0
.endm
#endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */
#endif /* __ASM_ASSEMBLER_H */

3
arch/arm64/include/asm/cpucaps.h
View File

@ -62,7 +62,8 @@
#define ARM64_HAS_ADDRESS_AUTH 52
#define ARM64_HAS_GENERIC_AUTH 53
#define ARM64_HAS_32BIT_EL1 54
#define ARM64_BTI 55
#define ARM64_NCAPS 55
#define ARM64_NCAPS 56
#endif /* __ASM_CPUCAPS_H */

5
arch/arm64/include/asm/cpufeature.h
View File

@ -687,6 +687,11 @@ static inline bool system_has_prio_mask_debugging(void)
system_uses_irq_prio_masking();
}
static inline bool system_supports_bti(void)
{
return IS_ENABLED(CONFIG_ARM64_BTI) && cpus_have_const_cap(ARM64_BTI);
}
#define ARM64_BP_HARDEN_UNKNOWN -1
#define ARM64_BP_HARDEN_WA_NEEDED 0
#define ARM64_BP_HARDEN_NOT_REQUIRED 1

50
arch/arm64/include/asm/elf.h
View File

@ -114,7 +114,11 @@
#ifndef __ASSEMBLY__
#include <uapi/linux/elf.h>
#include <linux/bug.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/types.h>
#include <asm/processor.h> /* for signal_minsigstksz, used by ARCH_DLINFO */
typedef unsigned long elf_greg_t;
@ -224,6 +228,52 @@ extern int aarch32_setup_additional_pages(struct linux_binprm *bprm,
#endif /* CONFIG_COMPAT */
struct arch_elf_state {
int flags;
};
#define ARM64_ELF_BTI (1 << 0)
#define INIT_ARCH_ELF_STATE { \
.flags = 0, \
}
static inline int arch_parse_elf_property(u32 type, const void *data,
size_t datasz, bool compat,
struct arch_elf_state *arch)
{
/* No known properties for AArch32 yet */
if (IS_ENABLED(CONFIG_COMPAT) && compat)
return 0;
if (type == GNU_PROPERTY_AARCH64_FEATURE_1_AND) {
const u32 *p = data;
if (datasz != sizeof(*p))
return -ENOEXEC;
if (system_supports_bti() &&
(*p & GNU_PROPERTY_AARCH64_FEATURE_1_BTI))
arch->flags |= ARM64_ELF_BTI;
}
return 0;
}
static inline int arch_elf_pt_proc(void *ehdr, void *phdr,
struct file *f, bool is_interp,
struct arch_elf_state *state)
{
return 0;
}
static inline int arch_check_elf(void *ehdr, bool has_interp,
void *interp_ehdr,
struct arch_elf_state *state)
{
return 0;
}
#endif /* !__ASSEMBLY__ */
#endif

2
arch/arm64/include/asm/esr.h
View File

@ -22,7 +22,7 @@
#define ESR_ELx_EC_PAC (0x09) /* EL2 and above */
/* Unallocated EC: 0x0A - 0x0B */
#define ESR_ELx_EC_CP14_64 (0x0C)
/* Unallocated EC: 0x0d */
#define ESR_ELx_EC_BTI (0x0D)
#define ESR_ELx_EC_ILL (0x0E)
/* Unallocated EC: 0x0F - 0x10 */
#define ESR_ELx_EC_SVC32 (0x11)

1
arch/arm64/include/asm/exception.h
View File

@ -34,6 +34,7 @@ static inline u32 disr_to_esr(u64 disr)
asmlinkage void enter_from_user_mode(void);
void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs);
void do_undefinstr(struct pt_regs *regs);
void do_bti(struct pt_regs *regs);
asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr);
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
struct pt_regs *regs);

1
arch/arm64/include/asm/hwcap.h
View File

@ -94,6 +94,7 @@
#define KERNEL_HWCAP_BF16 __khwcap2_feature(BF16)
#define KERNEL_HWCAP_DGH __khwcap2_feature(DGH)
#define KERNEL_HWCAP_RNG __khwcap2_feature(RNG)
#define KERNEL_HWCAP_BTI __khwcap2_feature(BTI)
/*
* This yields a mask that user programs can use to figure out what

30
arch/arm64/include/asm/insn.h
View File

@ -39,13 +39,37 @@ enum aarch64_insn_encoding_class {
* system instructions */
};
enum aarch64_insn_hint_op {
enum aarch64_insn_hint_cr_op {
AARCH64_INSN_HINT_NOP = 0x0 << 5,
AARCH64_INSN_HINT_YIELD = 0x1 << 5,
AARCH64_INSN_HINT_WFE = 0x2 << 5,
AARCH64_INSN_HINT_WFI = 0x3 << 5,
AARCH64_INSN_HINT_SEV = 0x4 << 5,
AARCH64_INSN_HINT_SEVL = 0x5 << 5,
AARCH64_INSN_HINT_XPACLRI = 0x07 << 5,
AARCH64_INSN_HINT_PACIA_1716 = 0x08 << 5,
AARCH64_INSN_HINT_PACIB_1716 = 0x0A << 5,
AARCH64_INSN_HINT_AUTIA_1716 = 0x0C << 5,
AARCH64_INSN_HINT_AUTIB_1716 = 0x0E << 5,
AARCH64_INSN_HINT_PACIAZ = 0x18 << 5,
AARCH64_INSN_HINT_PACIASP = 0x19 << 5,
AARCH64_INSN_HINT_PACIBZ = 0x1A << 5,
AARCH64_INSN_HINT_PACIBSP = 0x1B << 5,
AARCH64_INSN_HINT_AUTIAZ = 0x1C << 5,
AARCH64_INSN_HINT_AUTIASP = 0x1D << 5,
AARCH64_INSN_HINT_AUTIBZ = 0x1E << 5,
AARCH64_INSN_HINT_AUTIBSP = 0x1F << 5,
AARCH64_INSN_HINT_ESB = 0x10 << 5,
AARCH64_INSN_HINT_PSB = 0x11 << 5,
AARCH64_INSN_HINT_TSB = 0x12 << 5,
AARCH64_INSN_HINT_CSDB = 0x14 << 5,
AARCH64_INSN_HINT_BTI = 0x20 << 5,
AARCH64_INSN_HINT_BTIC = 0x22 << 5,
AARCH64_INSN_HINT_BTIJ = 0x24 << 5,
AARCH64_INSN_HINT_BTIJC = 0x26 << 5,
};
enum aarch64_insn_imm_type {
@ -344,7 +368,7 @@ __AARCH64_INSN_FUNCS(msr_reg, 0xFFF00000, 0xD5100000)
#undef __AARCH64_INSN_FUNCS
bool aarch64_insn_is_nop(u32 insn);
bool aarch64_insn_is_steppable_hint(u32 insn);
bool aarch64_insn_is_branch_imm(u32 insn);
static inline bool aarch64_insn_is_adr_adrp(u32 insn)
@ -370,7 +394,7 @@ u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,
enum aarch64_insn_branch_type type);
u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
enum aarch64_insn_condition cond);
u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_op op);
u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op);
u32 aarch64_insn_gen_nop(void);
u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
enum aarch64_insn_branch_type type);

6
arch/arm64/include/asm/kvm_emulate.h
View File

@ -507,10 +507,12 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
static __always_inline void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr)
{
if (vcpu_mode_is_32bit(vcpu))
if (vcpu_mode_is_32bit(vcpu)) {
kvm_skip_instr32(vcpu, is_wide_instr);
else
} else {
*vcpu_pc(vcpu) += 4;
*vcpu_cpsr(vcpu) &= ~PSR_BTYPE_MASK;
}
/* advance the singlestep state machine */
*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;

46
arch/arm64/include/asm/linkage.h
View File

@ -4,6 +4,52 @@
#define __ALIGN .align 2
#define __ALIGN_STR ".align 2"
#if defined(CONFIG_ARM64_BTI_KERNEL) && defined(__aarch64__)
/*
* Since current versions of gas reject the BTI instruction unless we
* set the architecture version to v8.5 we use the hint instruction
* instead.
*/
#define BTI_C hint 34 ;
#define BTI_J hint 36 ;
/*
* When using in-kernel BTI we need to ensure that PCS-conformant assembly
* functions have suitable annotations. Override SYM_FUNC_START to insert
* a BTI landing pad at the start of everything.
*/
#define SYM_FUNC_START(name) \
SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
BTI_C
#define SYM_FUNC_START_NOALIGN(name) \
SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \
BTI_C
#define SYM_FUNC_START_LOCAL(name) \
SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \
BTI_C
#define SYM_FUNC_START_LOCAL_NOALIGN(name) \
SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \
BTI_C
#define SYM_FUNC_START_WEAK(name) \
SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \
BTI_C
#define SYM_FUNC_START_WEAK_NOALIGN(name) \
SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \
BTI_C
#define SYM_INNER_LABEL(name, linkage) \
.type name SYM_T_NONE ASM_NL \
SYM_ENTRY(name, linkage, SYM_A_NONE) \
BTI_J
#endif
/*
* Annotate a function as position independent, i.e., safe to be called before
* the kernel virtual mapping is activated.

37
arch/arm64/include/asm/mman.h Normal file
View File

@ -0,0 +1,37 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_MMAN_H__
#define __ASM_MMAN_H__
#include <linux/compiler.h>
#include <linux/types.h>
#include <uapi/asm/mman.h>
static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot,
unsigned long pkey __always_unused)
{
if (system_supports_bti() && (prot & PROT_BTI))
return VM_ARM64_BTI;
return 0;
}
#define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey)
static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags)
{
return (vm_flags & VM_ARM64_BTI) ? __pgprot(PTE_GP) : __pgprot(0);
}
#define arch_vm_get_page_prot(vm_flags) arch_vm_get_page_prot(vm_flags)
static inline bool arch_validate_prot(unsigned long prot,
unsigned long addr __always_unused)
{
unsigned long supported = PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM;
if (system_supports_bti())
supported |= PROT_BTI;
return (prot & ~supported) == 0;
}
#define arch_validate_prot(prot, addr) arch_validate_prot(prot, addr)
#endif /* ! __ASM_MMAN_H__ */

1
arch/arm64/include/asm/pgtable-hwdef.h
View File

@ -151,6 +151,7 @@
#define PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
#define PTE_AF (_AT(pteval_t, 1) << 10) /* Access Flag */
#define PTE_NG (_AT(pteval_t, 1) << 11) /* nG */
#define PTE_GP (_AT(pteval_t, 1) << 50) /* BTI guarded */
#define PTE_DBM (_AT(pteval_t, 1) << 51) /* Dirty Bit Management */
#define PTE_CONT (_AT(pteval_t, 1) << 52) /* Contiguous range */
#define PTE_PXN (_AT(pteval_t, 1) << 53) /* Privileged XN */

11
arch/arm64/include/asm/pgtable-prot.h
View File

@ -21,6 +21,7 @@
#ifndef __ASSEMBLY__
#include <asm/cpufeature.h>
#include <asm/pgtable-types.h>
extern bool arm64_use_ng_mappings;
@ -31,6 +32,16 @@ extern bool arm64_use_ng_mappings;
#define PTE_MAYBE_NG (arm64_use_ng_mappings ? PTE_NG : 0)
#define PMD_MAYBE_NG (arm64_use_ng_mappings ? PMD_SECT_NG : 0)
/*
* If we have userspace only BTI we don't want to mark kernel pages
* guarded even if the system does support BTI.
*/
#ifdef CONFIG_ARM64_BTI_KERNEL
#define PTE_MAYBE_GP (system_supports_bti() ? PTE_GP : 0)
#else
#define PTE_MAYBE_GP 0
#endif
#define PROT_DEFAULT (_PROT_DEFAULT | PTE_MAYBE_NG)
#define PROT_SECT_DEFAULT (_PROT_SECT_DEFAULT | PMD_MAYBE_NG)

2
arch/arm64/include/asm/pgtable.h
View File

@ -661,7 +661,7 @@ static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
PTE_PROT_NONE | PTE_VALID | PTE_WRITE;
PTE_PROT_NONE | PTE_VALID | PTE_WRITE | PTE_GP;
/* preserve the hardware dirty information */
if (pte_hw_dirty(pte))
pte = pte_mkdirty(pte);

1
arch/arm64/include/asm/ptrace.h
View File

@ -35,6 +35,7 @@
#define GIC_PRIO_PSR_I_SET (1 << 4)
/* Additional SPSR bits not exposed in the UABI */
#define PSR_IL_BIT (1 << 20)
/* AArch32-specific ptrace requests */

3
arch/arm64/include/asm/sysreg.h
View File

@ -559,6 +559,8 @@
#endif
/* SCTLR_EL1 specific flags. */
#define SCTLR_EL1_BT1 (BIT(36))
#define SCTLR_EL1_BT0 (BIT(35))
#define SCTLR_EL1_UCI (BIT(26))
#define SCTLR_EL1_E0E (BIT(24))
#define SCTLR_EL1_SPAN (BIT(23))
@ -679,6 +681,7 @@
#define ID_AA64PFR1_SSBS_PSTATE_NI 0
#define ID_AA64PFR1_SSBS_PSTATE_ONLY 1
#define ID_AA64PFR1_SSBS_PSTATE_INSNS 2
#define ID_AA64PFR1_BT_BTI 0x1
/* id_aa64zfr0 */
#define ID_AA64ZFR0_F64MM_SHIFT 56

1
arch/arm64/include/uapi/asm/hwcap.h
View File

@ -73,5 +73,6 @@
#define HWCAP2_BF16 (1 << 14)
#define HWCAP2_DGH (1 << 15)
#define HWCAP2_RNG (1 << 16)
#define HWCAP2_BTI (1 << 17)
#endif /* _UAPI__ASM_HWCAP_H */

9
arch/arm64/include/uapi/asm/mman.h Normal file
View File

@ -0,0 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
#ifndef _UAPI__ASM_MMAN_H
#define _UAPI__ASM_MMAN_H
#include <asm-generic/mman.h>
#define PROT_BTI 0x10 /* BTI guarded page */
#endif /* ! _UAPI__ASM_MMAN_H */

9
arch/arm64/include/uapi/asm/ptrace.h
View File

@ -46,6 +46,7 @@
#define PSR_I_BIT 0x00000080
#define PSR_A_BIT 0x00000100
#define PSR_D_BIT 0x00000200
#define PSR_BTYPE_MASK 0x00000c00
#define PSR_SSBS_BIT 0x00001000
#define PSR_PAN_BIT 0x00400000
#define PSR_UAO_BIT 0x00800000
@ -55,6 +56,8 @@
#define PSR_Z_BIT 0x40000000
#define PSR_N_BIT 0x80000000
#define PSR_BTYPE_SHIFT 10
/*
* Groups of PSR bits
*/
@ -63,6 +66,12 @@
#define PSR_x 0x0000ff00 /* Extension */
#define PSR_c 0x000000ff /* Control */
/* Convenience names for the values of PSTATE.BTYPE */
#define PSR_BTYPE_NONE (0b00 << PSR_BTYPE_SHIFT)
#define PSR_BTYPE_JC (0b01 << PSR_BTYPE_SHIFT)
#define PSR_BTYPE_C (0b10 << PSR_BTYPE_SHIFT)
#define PSR_BTYPE_J (0b11 << PSR_BTYPE_SHIFT)
/* syscall emulation path in ptrace */
#define PTRACE_SYSEMU 31
#define PTRACE_SYSEMU_SINGLESTEP 32

4
arch/arm64/kernel/cpu-reset.S
View File

@ -29,7 +29,7 @@
* branch to what would be the reset vector. It must be executed with the
* flat identity mapping.
*/
ENTRY(__cpu_soft_restart)
SYM_CODE_START(__cpu_soft_restart)
/* Clear sctlr_el1 flags. */
mrs x12, sctlr_el1
mov_q x13, SCTLR_ELx_FLAGS
@ -47,6 +47,6 @@ ENTRY(__cpu_soft_restart)
mov x1, x3 // arg1
mov x2, x4 // arg2
br x8
ENDPROC(__cpu_soft_restart)
SYM_CODE_END(__cpu_soft_restart)
.popsection

37
arch/arm64/kernel/cpufeature.c
View File

@ -241,6 +241,8 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MPAMFRAC_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_RASFRAC_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_BTI),
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_BT_SHIFT, 4, 0),
ARM64_FTR_END,
};
@ -1638,6 +1640,21 @@ static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry,
}
#endif
#ifdef CONFIG_ARM64_BTI
static void bti_enable(const struct arm64_cpu_capabilities *__unused)
{
/*
* Use of X16/X17 for tail-calls and trampolines that jump to
* function entry points using BR is a requirement for
* marking binaries with GNU_PROPERTY_AARCH64_FEATURE_1_BTI.
* So, be strict and forbid other BRs using other registers to
* jump onto a PACIxSP instruction:
*/
sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1);
isb();
}
#endif /* CONFIG_ARM64_BTI */
/* Internal helper functions to match cpu capability type */
static bool
cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
@ -2019,6 +2036,23 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.sign = FTR_UNSIGNED,
.min_field_value = 1,
},
#endif
#ifdef CONFIG_ARM64_BTI
{
.desc = "Branch Target Identification",
.capability = ARM64_BTI,
#ifdef CONFIG_ARM64_BTI_KERNEL
.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
#else
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
#endif
.matches = has_cpuid_feature,
.cpu_enable = bti_enable,
.sys_reg = SYS_ID_AA64PFR1_EL1,
.field_pos = ID_AA64PFR1_BT_SHIFT,
.min_field_value = ID_AA64PFR1_BT_BTI,
.sign = FTR_UNSIGNED,
},
#endif
{},
};
@ -2129,6 +2163,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_F64MM_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_F64MM, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM),
#endif
HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, KERNEL_HWCAP_SSBS),
#ifdef CONFIG_ARM64_BTI
HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_BT_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_BT_BTI, CAP_HWCAP, KERNEL_HWCAP_BTI),
#endif
#ifdef CONFIG_ARM64_PTR_AUTH
HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, KERNEL_HWCAP_PACA),
HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, KERNEL_HWCAP_PACG),

1
arch/arm64/kernel/cpuinfo.c
View File

@ -92,6 +92,7 @@ static const char *const hwcap_str[] = {
"bf16",
"dgh",
"rng",
"bti",
NULL
};

4
arch/arm64/kernel/efi-rt-wrapper.S
View File

@ -5,7 +5,7 @@
#include <linux/linkage.h>
ENTRY(__efi_rt_asm_wrapper)
SYM_FUNC_START(__efi_rt_asm_wrapper)
stp x29, x30, [sp, #-32]!
mov x29, sp
@ -35,4 +35,4 @@ ENTRY(__efi_rt_asm_wrapper)
b.ne 0f
ret
0: b efi_handle_corrupted_x18 // tail call
ENDPROC(__efi_rt_asm_wrapper)
SYM_FUNC_END(__efi_rt_asm_wrapper)

11
arch/arm64/kernel/entry-common.c
View File

@ -188,6 +188,14 @@ static void notrace el0_undef(struct pt_regs *regs)
}
NOKPROBE_SYMBOL(el0_undef);
static void notrace el0_bti(struct pt_regs *regs)
{
user_exit_irqoff();
local_daif_restore(DAIF_PROCCTX);
do_bti(regs);
}
NOKPROBE_SYMBOL(el0_bti);
static void notrace el0_inv(struct pt_regs *regs, unsigned long esr)
{
user_exit_irqoff();
@ -255,6 +263,9 @@ asmlinkage void notrace el0_sync_handler(struct pt_regs *regs)
case ESR_ELx_EC_UNKNOWN:
el0_undef(regs);
break;
case ESR_ELx_EC_BTI:
el0_bti(regs);
break;
case ESR_ELx_EC_BREAKPT_LOW:
case ESR_ELx_EC_SOFTSTP_LOW:
case ESR_ELx_EC_WATCHPT_LOW:

20
arch/arm64/kernel/entry-fpsimd.S
View File

@ -16,34 +16,34 @@
*
* x0 - pointer to struct fpsimd_state
*/
ENTRY(fpsimd_save_state)
SYM_FUNC_START(fpsimd_save_state)
fpsimd_save x0, 8
ret
ENDPROC(fpsimd_save_state)
SYM_FUNC_END(fpsimd_save_state)
/*
* Load the FP registers.
*
* x0 - pointer to struct fpsimd_state
*/
ENTRY(fpsimd_load_state)
SYM_FUNC_START(fpsimd_load_state)
fpsimd_restore x0, 8
ret
ENDPROC(fpsimd_load_state)
SYM_FUNC_END(fpsimd_load_state)
#ifdef CONFIG_ARM64_SVE
ENTRY(sve_save_state)
SYM_FUNC_START(sve_save_state)
sve_save 0, x1, 2
ret
ENDPROC(sve_save_state)
SYM_FUNC_END(sve_save_state)
ENTRY(sve_load_state)
SYM_FUNC_START(sve_load_state)
sve_load 0, x1, x2, 3, x4
ret
ENDPROC(sve_load_state)
SYM_FUNC_END(sve_load_state)
ENTRY(sve_get_vl)
SYM_FUNC_START(sve_get_vl)
_sve_rdvl 0, 1
ret
ENDPROC(sve_get_vl)
SYM_FUNC_END(sve_get_vl)
#endif /* CONFIG_ARM64_SVE */

27
arch/arm64/kernel/entry.S
View File

@ -727,21 +727,10 @@ el0_error_naked:
b ret_to_user
SYM_CODE_END(el0_error)
/*
* Ok, we need to do extra processing, enter the slow path.
*/
work_pending:
mov x0, sp // 'regs'
bl do_notify_resume
#ifdef CONFIG_TRACE_IRQFLAGS
bl trace_hardirqs_on // enabled while in userspace
#endif
ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for single-step
b finish_ret_to_user
/*
* "slow" syscall return path.
*/
ret_to_user:
SYM_CODE_START_LOCAL(ret_to_user)
disable_daif
gic_prio_kentry_setup tmp=x3
ldr x1, [tsk, #TSK_TI_FLAGS]
@ -753,7 +742,19 @@ finish_ret_to_user:
bl stackleak_erase
#endif
kernel_exit 0
ENDPROC(ret_to_user)
/*
* Ok, we need to do extra processing, enter the slow path.
*/
work_pending:
mov x0, sp // 'regs'
bl do_notify_resume
#ifdef CONFIG_TRACE_IRQFLAGS
bl trace_hardirqs_on // enabled while in userspace
#endif
ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for single-step
b finish_ret_to_user
SYM_CODE_END(ret_to_user)
.popsection // .entry.text

16
arch/arm64/kernel/hibernate-asm.S
View File

@ -65,7 +65,7 @@
* x5: physical address of a zero page that remains zero after resume
*/
.pushsection ".hibernate_exit.text", "ax"
ENTRY(swsusp_arch_suspend_exit)
SYM_CODE_START(swsusp_arch_suspend_exit)
/*
* We execute from ttbr0, change ttbr1 to our copied linear map tables
* with a break-before-make via the zero page
@ -110,7 +110,7 @@ ENTRY(swsusp_arch_suspend_exit)
cbz x24, 3f /* Do we need to re-initialise EL2? */
hvc #0
3: ret
ENDPROC(swsusp_arch_suspend_exit)
SYM_CODE_END(swsusp_arch_suspend_exit)
/*
* Restore the hyp stub.
@ -119,15 +119,15 @@ ENDPROC(swsusp_arch_suspend_exit)
*
* x24: The physical address of __hyp_stub_vectors
*/
el1_sync:
SYM_CODE_START_LOCAL(el1_sync)
msr vbar_el2, x24
eret
ENDPROC(el1_sync)
SYM_CODE_END(el1_sync)
.macro invalid_vector label
\label:
SYM_CODE_START_LOCAL(\label)
b \label
ENDPROC(\label)
SYM_CODE_END(\label)
.endm
invalid_vector el2_sync_invalid
@ -141,7 +141,7 @@ ENDPROC(\label)
/* el2 vectors - switch el2 here while we restore the memory image. */
.align 11
ENTRY(hibernate_el2_vectors)
SYM_CODE_START(hibernate_el2_vectors)
ventry el2_sync_invalid // Synchronous EL2t
ventry el2_irq_invalid // IRQ EL2t
ventry el2_fiq_invalid // FIQ EL2t
@ -161,6 +161,6 @@ ENTRY(hibernate_el2_vectors)
ventry el1_irq_invalid // IRQ 32-bit EL1
ventry el1_fiq_invalid // FIQ 32-bit EL1
ventry el1_error_invalid // Error 32-bit EL1
END(hibernate_el2_vectors)
SYM_CODE_END(hibernate_el2_vectors)
.popsection

20
arch/arm64/kernel/hyp-stub.S
View File

@ -21,7 +21,7 @@
.align 11
ENTRY(__hyp_stub_vectors)
SYM_CODE_START(__hyp_stub_vectors)
ventry el2_sync_invalid // Synchronous EL2t
ventry el2_irq_invalid // IRQ EL2t
ventry el2_fiq_invalid // FIQ EL2t
@ -41,11 +41,11 @@ ENTRY(__hyp_stub_vectors)
ventry el1_irq_invalid // IRQ 32-bit EL1
ventry el1_fiq_invalid // FIQ 32-bit EL1
ventry el1_error_invalid // Error 32-bit EL1
ENDPROC(__hyp_stub_vectors)
SYM_CODE_END(__hyp_stub_vectors)
.align 11
el1_sync:
SYM_CODE_START_LOCAL(el1_sync)
cmp x0, #HVC_SET_VECTORS
b.ne 2f
msr vbar_el2, x1
@ -68,12 +68,12 @@ el1_sync:
9: mov x0, xzr
eret
ENDPROC(el1_sync)
SYM_CODE_END(el1_sync)
.macro invalid_vector label
\label:
SYM_CODE_START_LOCAL(\label)
b \label
ENDPROC(\label)
SYM_CODE_END(\label)
.endm
invalid_vector el2_sync_invalid
@ -106,15 +106,15 @@ ENDPROC(\label)
* initialisation entry point.
*/
ENTRY(__hyp_set_vectors)
SYM_FUNC_START(__hyp_set_vectors)
mov x1, x0
mov x0, #HVC_SET_VECTORS
hvc #0
ret
ENDPROC(__hyp_set_vectors)
SYM_FUNC_END(__hyp_set_vectors)
ENTRY(__hyp_reset_vectors)
SYM_FUNC_START(__hyp_reset_vectors)
mov x0, #HVC_RESET_VECTORS
hvc #0
ret
ENDPROC(__hyp_reset_vectors)
SYM_FUNC_END(__hyp_reset_vectors)

32
arch/arm64/kernel/insn.c
View File

@ -51,21 +51,33 @@ enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
}
/* NOP is an alias of HINT */
bool __kprobes aarch64_insn_is_nop(u32 insn)
bool __kprobes aarch64_insn_is_steppable_hint(u32 insn)
{
if (!aarch64_insn_is_hint(insn))
return false;
switch (insn & 0xFE0) {
case AARCH64_INSN_HINT_YIELD:
case AARCH64_INSN_HINT_WFE:
case AARCH64_INSN_HINT_WFI:
case AARCH64_INSN_HINT_SEV:
case AARCH64_INSN_HINT_SEVL:
return false;
default:
case AARCH64_INSN_HINT_XPACLRI:
case AARCH64_INSN_HINT_PACIA_1716:
case AARCH64_INSN_HINT_PACIB_1716:
case AARCH64_INSN_HINT_AUTIA_1716:
case AARCH64_INSN_HINT_AUTIB_1716:
case AARCH64_INSN_HINT_PACIAZ:
case AARCH64_INSN_HINT_PACIASP:
case AARCH64_INSN_HINT_PACIBZ:
case AARCH64_INSN_HINT_PACIBSP:
case AARCH64_INSN_HINT_AUTIAZ:
case AARCH64_INSN_HINT_AUTIASP:
case AARCH64_INSN_HINT_AUTIBZ:
case AARCH64_INSN_HINT_AUTIBSP:
case AARCH64_INSN_HINT_BTI:
case AARCH64_INSN_HINT_BTIC:
case AARCH64_INSN_HINT_BTIJ:
case AARCH64_INSN_HINT_BTIJC:
case AARCH64_INSN_HINT_NOP:
return true;
default:
return false;
}
}
@ -574,7 +586,7 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
offset >> 2);
}
u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op)
{
return aarch64_insn_get_hint_value() | op;
}

2
arch/arm64/kernel/probes/decode-insn.c
View File

@ -46,7 +46,7 @@ static bool __kprobes aarch64_insn_is_steppable(u32 insn)
* except for the NOP case.
*/
if (aarch64_insn_is_hint(insn))
return aarch64_insn_is_nop(insn);
return aarch64_insn_is_steppable_hint(insn);
return true;
}

4
arch/arm64/kernel/probes/kprobes_trampoline.S
View File

@ -61,7 +61,7 @@
ldp x28, x29, [sp, #S_X28]
.endm
ENTRY(kretprobe_trampoline)
SYM_CODE_START(kretprobe_trampoline)
sub sp, sp, #S_FRAME_SIZE
save_all_base_regs
@ -79,4 +79,4 @@ ENTRY(kretprobe_trampoline)
add sp, sp, #S_FRAME_SIZE
ret
ENDPROC(kretprobe_trampoline)
SYM_CODE_END(kretprobe_trampoline)

41
arch/arm64/kernel/process.c
View File

@ -11,6 +11,7 @@
#include <linux/compat.h>
#include <linux/efi.h>
#include <linux/elf.h>
#include <linux/export.h>
#include <linux/sched.h>
#include <linux/sched/debug.h>
@ -18,6 +19,7 @@
#include <linux/sched/task_stack.h>
#include <linux/kernel.h>
#include <linux/lockdep.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/stddef.h>
#include <linux/sysctl.h>
@ -209,6 +211,15 @@ void machine_restart(char *cmd)
while (1);
}
#define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = str
static const char *const btypes[] = {
bstr(NONE, "--"),
bstr( JC, "jc"),
bstr( C, "-c"),
bstr( J , "j-")
};
#undef bstr
static void print_pstate(struct pt_regs *regs)
{
u64 pstate = regs->pstate;
@ -227,7 +238,10 @@ static void print_pstate(struct pt_regs *regs)
pstate & PSR_AA32_I_BIT ? 'I' : 'i',
pstate & PSR_AA32_F_BIT ? 'F' : 'f');
} else {
printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO)\n",
const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >>
PSR_BTYPE_SHIFT];
printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO BTYPE=%s)\n",
pstate,
pstate & PSR_N_BIT ? 'N' : 'n',
pstate & PSR_Z_BIT ? 'Z' : 'z',
@ -238,7 +252,8 @@ static void print_pstate(struct pt_regs *regs)
pstate & PSR_I_BIT ? 'I' : 'i',
pstate & PSR_F_BIT ? 'F' : 'f',
pstate & PSR_PAN_BIT ? '+' : '-',
pstate & PSR_UAO_BIT ? '+' : '-');
pstate & PSR_UAO_BIT ? '+' : '-',
btype_str);
}
}
@ -655,3 +670,25 @@ asmlinkage void __sched arm64_preempt_schedule_irq(void)
if (system_capabilities_finalized())
preempt_schedule_irq();
}
#ifdef CONFIG_BINFMT_ELF
int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state,
bool has_interp, bool is_interp)
{
/*
* For dynamically linked executables the interpreter is
* responsible for setting PROT_BTI on everything except
* itself.
*/
if (is_interp != has_interp)
return prot;
if (!(state->flags & ARM64_ELF_BTI))
return prot;
if (prot & PROT_EXEC)
prot |= PROT_BTI;
return prot;
}
#endif

2
arch/arm64/kernel/ptrace.c
View File

@ -1874,7 +1874,7 @@ void syscall_trace_exit(struct pt_regs *regs)
*/
#define SPSR_EL1_AARCH64_RES0_BITS \
(GENMASK_ULL(63, 32) | GENMASK_ULL(27, 25) | GENMASK_ULL(23, 22) | \
GENMASK_ULL(20, 13) | GENMASK_ULL(11, 10) | GENMASK_ULL(5, 5))
GENMASK_ULL(20, 13) | GENMASK_ULL(5, 5))
#define SPSR_EL1_AARCH32_RES0_BITS \
(GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20))

44
arch/arm64/kernel/reloc_test_syms.S
View File

@ -5,81 +5,81 @@
#include <linux/linkage.h>
ENTRY(absolute_data64)
SYM_FUNC_START(absolute_data64)
ldr x0, 0f
ret
0: .quad sym64_abs
ENDPROC(absolute_data64)
SYM_FUNC_END(absolute_data64)
ENTRY(absolute_data32)
SYM_FUNC_START(absolute_data32)
ldr w0, 0f
ret
0: .long sym32_abs
ENDPROC(absolute_data32)
SYM_FUNC_END(absolute_data32)
ENTRY(absolute_data16)
SYM_FUNC_START(absolute_data16)
adr x0, 0f
ldrh w0, [x0]
ret
0: .short sym16_abs, 0
ENDPROC(absolute_data16)
SYM_FUNC_END(absolute_data16)
ENTRY(signed_movw)
SYM_FUNC_START(signed_movw)
movz x0, #:abs_g2_s:sym64_abs
movk x0, #:abs_g1_nc:sym64_abs
movk x0, #:abs_g0_nc:sym64_abs
ret
ENDPROC(signed_movw)
SYM_FUNC_END(signed_movw)
ENTRY(unsigned_movw)
SYM_FUNC_START(unsigned_movw)
movz x0, #:abs_g3:sym64_abs
movk x0, #:abs_g2_nc:sym64_abs
movk x0, #:abs_g1_nc:sym64_abs
movk x0, #:abs_g0_nc:sym64_abs
ret
ENDPROC(unsigned_movw)
SYM_FUNC_END(unsigned_movw)
.align 12
.space 0xff8
ENTRY(relative_adrp)
SYM_FUNC_START(relative_adrp)
adrp x0, sym64_rel
add x0, x0, #:lo12:sym64_rel
ret
ENDPROC(relative_adrp)
SYM_FUNC_END(relative_adrp)
.align 12
.space 0xffc
ENTRY(relative_adrp_far)
SYM_FUNC_START(relative_adrp_far)
adrp x0, memstart_addr
add x0, x0, #:lo12:memstart_addr
ret
ENDPROC(relative_adrp_far)
SYM_FUNC_END(relative_adrp_far)
ENTRY(relative_adr)
SYM_FUNC_START(relative_adr)
adr x0, sym64_rel
ret
ENDPROC(relative_adr)
SYM_FUNC_END(relative_adr)
ENTRY(relative_data64)
SYM_FUNC_START(relative_data64)
adr x1, 0f
ldr x0, [x1]
add x0, x0, x1
ret
0: .quad sym64_rel - .
ENDPROC(relative_data64)
SYM_FUNC_END(relative_data64)
ENTRY(relative_data32)
SYM_FUNC_START(relative_data32)
adr x1, 0f
ldr w0, [x1]
add x0, x0, x1
ret
0: .long sym64_rel - .
ENDPROC(relative_data32)
SYM_FUNC_END(relative_data32)
ENTRY(relative_data16)
SYM_FUNC_START(relative_data16)
adr x1, 0f
ldrsh w0, [x1]
add x0, x0, x1
ret
0: .short sym64_rel - ., 0
ENDPROC(relative_data16)
SYM_FUNC_END(relative_data16)

4
arch/arm64/kernel/relocate_kernel.S
View File

@ -26,7 +26,7 @@
* control_code_page, a special page which has been set up to be preserved
* during the copy operation.
*/
ENTRY(arm64_relocate_new_kernel)
SYM_CODE_START(arm64_relocate_new_kernel)
/* Setup the list loop variables. */
mov x18, x2 /* x18 = dtb address */
@ -111,7 +111,7 @@ ENTRY(arm64_relocate_new_kernel)
mov x3, xzr
br x17
ENDPROC(arm64_relocate_new_kernel)
SYM_CODE_END(arm64_relocate_new_kernel)
.align 3 /* To keep the 64-bit values below naturally aligned. */

16
arch/arm64/kernel/signal.c
View File

@ -732,6 +732,22 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka,
regs->regs[29] = (unsigned long)&user->next_frame->fp;
regs->pc = (unsigned long)ka->sa.sa_handler;
/*
* Signal delivery is a (wacky) indirect function call in
* userspace, so simulate the same setting of BTYPE as a BLR
* <register containing the signal handler entry point>.
* Signal delivery to a location in a PROT_BTI guarded page
* that is not a function entry point will now trigger a
* SIGILL in userspace.
*
* If the signal handler entry point is not in a PROT_BTI
* guarded page, this is harmless.
*/
if (system_supports_bti()) {
regs->pstate &= ~PSR_BTYPE_MASK;
regs->pstate |= PSR_BTYPE_C;
}
if (ka->sa.sa_flags & SA_RESTORER)
sigtramp = ka->sa.sa_restorer;
else

12
arch/arm64/kernel/sleep.S
View File

@ -62,7 +62,7 @@
*
* x0 = struct sleep_stack_data area
*/
ENTRY(__cpu_suspend_enter)
SYM_FUNC_START(__cpu_suspend_enter)
stp x29, lr, [x0, #SLEEP_STACK_DATA_CALLEE_REGS]
stp x19, x20, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+16]
stp x21, x22, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+32]
@ -95,10 +95,10 @@ ENTRY(__cpu_suspend_enter)
ldp x29, lr, [sp], #16
mov x0, #1
ret
ENDPROC(__cpu_suspend_enter)
SYM_FUNC_END(__cpu_suspend_enter)
.pushsection ".idmap.text", "awx"
ENTRY(cpu_resume)
SYM_CODE_START(cpu_resume)
bl el2_setup // if in EL2 drop to EL1 cleanly
bl __cpu_setup
/* enable the MMU early - so we can access sleep_save_stash by va */
@ -106,11 +106,11 @@ ENTRY(cpu_resume)
bl __enable_mmu
ldr x8, =_cpu_resume
br x8
ENDPROC(cpu_resume)
SYM_CODE_END(cpu_resume)
.ltorg
.popsection
ENTRY(_cpu_resume)
SYM_FUNC_START(_cpu_resume)
mrs x1, mpidr_el1
adr_l x8, mpidr_hash // x8 = struct mpidr_hash virt address
@ -146,4 +146,4 @@ ENTRY(_cpu_resume)
ldp x29, lr, [x29]
mov x0, #0
ret
ENDPROC(_cpu_resume)
SYM_FUNC_END(_cpu_resume)

8
arch/arm64/kernel/smccc-call.S
View File

@ -30,9 +30,9 @@
* unsigned long a6, unsigned long a7, struct arm_smccc_res *res,
* struct arm_smccc_quirk *quirk)
*/
ENTRY(__arm_smccc_smc)
SYM_FUNC_START(__arm_smccc_smc)
SMCCC smc
ENDPROC(__arm_smccc_smc)
SYM_FUNC_END(__arm_smccc_smc)
EXPORT_SYMBOL(__arm_smccc_smc)
/*
@ -41,7 +41,7 @@ EXPORT_SYMBOL(__arm_smccc_smc)
* unsigned long a6, unsigned long a7, struct arm_smccc_res *res,
* struct arm_smccc_quirk *quirk)
*/
ENTRY(__arm_smccc_hvc)
SYM_FUNC_START(__arm_smccc_hvc)
SMCCC hvc
ENDPROC(__arm_smccc_hvc)
SYM_FUNC_END(__arm_smccc_hvc)
EXPORT_SYMBOL(__arm_smccc_hvc)

18
arch/arm64/kernel/syscall.c
View File

@ -98,6 +98,24 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
regs->orig_x0 = regs->regs[0];
regs->syscallno = scno;
/*
* BTI note:
* The architecture does not guarantee that SPSR.BTYPE is zero
* on taking an SVC, so we could return to userspace with a
* non-zero BTYPE after the syscall.
*
* This shouldn't matter except when userspace is explicitly
* doing something stupid, such as setting PROT_BTI on a page
* that lacks conforming BTI/PACIxSP instructions, falling
* through from one executable page to another with differing
* PROT_BTI, or messing with BTYPE via ptrace: in such cases,
* userspace should not be surprised if a SIGILL occurs on
* syscall return.
*
* So, don't touch regs->pstate & PSR_BTYPE_MASK here.
* (Similarly for HVC and SMC elsewhere.)
*/
cortex_a76_erratum_1463225_svc_handler();
local_daif_restore(DAIF_PROCCTX);
user_exit();

131
arch/arm64/kernel/traps.c
View File

@ -272,6 +272,61 @@ void arm64_notify_die(const char *str, struct pt_regs *regs,
}
}
#ifdef CONFIG_COMPAT
#define PSTATE_IT_1_0_SHIFT 25
#define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT)
#define PSTATE_IT_7_2_SHIFT 10
#define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT)
static u32 compat_get_it_state(struct pt_regs *regs)
{
u32 it, pstate = regs->pstate;
it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT;
it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2;
return it;
}
static void compat_set_it_state(struct pt_regs *regs, u32 it)
{
u32 pstate_it;
pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK;
pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK;
regs->pstate &= ~PSR_AA32_IT_MASK;
regs->pstate |= pstate_it;
}
static void advance_itstate(struct pt_regs *regs)
{
u32 it;
/* ARM mode */
if (!(regs->pstate & PSR_AA32_T_BIT) ||
!(regs->pstate & PSR_AA32_IT_MASK))
return;
it = compat_get_it_state(regs);
/*
* If this is the last instruction of the block, wipe the IT
* state. Otherwise advance it.
*/
if (!(it & 7))
it = 0;
else
it = (it & 0xe0) | ((it << 1) & 0x1f);
compat_set_it_state(regs, it);
}
#else
static void advance_itstate(struct pt_regs *regs)
{
}
#endif
void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
{
regs->pc += size;
@ -282,6 +337,11 @@ void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
*/
if (user_mode(regs))
user_fastforward_single_step(current);
if (compat_user_mode(regs))
advance_itstate(regs);
else
regs->pstate &= ~PSR_BTYPE_MASK;
}
static LIST_HEAD(undef_hook);
@ -411,6 +471,13 @@ void do_undefinstr(struct pt_regs *regs)
}
NOKPROBE_SYMBOL(do_undefinstr);
void do_bti(struct pt_regs *regs)
{
BUG_ON(!user_mode(regs));
force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
}
NOKPROBE_SYMBOL(do_bti);
#define __user_cache_maint(insn, address, res) \
if (address >= user_addr_max()) { \
res = -EFAULT; \
@ -566,34 +633,7 @@ static const struct sys64_hook sys64_hooks[] = {
{},
};
#ifdef CONFIG_COMPAT
#define PSTATE_IT_1_0_SHIFT 25
#define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT)
#define PSTATE_IT_7_2_SHIFT 10
#define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT)
static u32 compat_get_it_state(struct pt_regs *regs)
{
u32 it, pstate = regs->pstate;
it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT;
it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2;
return it;
}
static void compat_set_it_state(struct pt_regs *regs, u32 it)
{
u32 pstate_it;
pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK;
pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK;
regs->pstate &= ~PSR_AA32_IT_MASK;
regs->pstate |= pstate_it;
}
static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs)
{
int cond;
@ -614,42 +654,12 @@ static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs)
return aarch32_opcode_cond_checks[cond](regs->pstate);
}
static void advance_itstate(struct pt_regs *regs)
{
u32 it;
/* ARM mode */
if (!(regs->pstate & PSR_AA32_T_BIT) ||
!(regs->pstate & PSR_AA32_IT_MASK))
return;
it = compat_get_it_state(regs);
/*
* If this is the last instruction of the block, wipe the IT
* state. Otherwise advance it.
*/
if (!(it & 7))
it = 0;
else
it = (it & 0xe0) | ((it << 1) & 0x1f);
compat_set_it_state(regs, it);
}
static void arm64_compat_skip_faulting_instruction(struct pt_regs *regs,
unsigned int sz)
{
advance_itstate(regs);
arm64_skip_faulting_instruction(regs, sz);
}
static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
{
int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT;
pt_regs_write_reg(regs, reg, arch_timer_get_rate());
arm64_compat_skip_faulting_instruction(regs, 4);
arm64_skip_faulting_instruction(regs, 4);
}
static const struct sys64_hook cp15_32_hooks[] = {
@ -669,7 +679,7 @@ static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
pt_regs_write_reg(regs, rt, lower_32_bits(val));
pt_regs_write_reg(regs, rt2, upper_32_bits(val));
arm64_compat_skip_faulting_instruction(regs, 4);
arm64_skip_faulting_instruction(regs, 4);
}
static const struct sys64_hook cp15_64_hooks[] = {
@ -690,7 +700,7 @@ void do_cp15instr(unsigned int esr, struct pt_regs *regs)
* There is no T16 variant of a CP access, so we
* always advance PC by 4 bytes.
*/
arm64_compat_skip_faulting_instruction(regs, 4);
arm64_skip_faulting_instruction(regs, 4);
return;
}
@ -753,6 +763,7 @@ static const char *esr_class_str[] = {
[ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS",
[ESR_ELx_EC_PAC] = "PAC",
[ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC",
[ESR_ELx_EC_BTI] = "BTI",
[ESR_ELx_EC_ILL] = "PSTATE.IL",
[ESR_ELx_EC_SVC32] = "SVC (AArch32)",
[ESR_ELx_EC_HVC32] = "HVC (AArch32)",

6
arch/arm64/kernel/vdso.c
View File

@ -136,6 +136,7 @@ static int __setup_additional_pages(enum vdso_abi abi,
int uses_interp)
{
unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
unsigned long gp_flags = 0;
void *ret;
vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;
@ -154,10 +155,13 @@ static int __setup_additional_pages(enum vdso_abi abi,
if (IS_ERR(ret))
goto up_fail;
if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) && system_supports_bti())
gp_flags = VM_ARM64_BTI;
vdso_base += PAGE_SIZE;
mm->context.vdso = (void *)vdso_base;
ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
VM_READ|VM_EXEC|
VM_READ|VM_EXEC|gp_flags|
VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
vdso_info[abi].cm);
if (IS_ERR(ret))

4
arch/arm64/kernel/vdso/Makefile
View File

@ -17,12 +17,14 @@ obj-vdso := vgettimeofday.o note.o sigreturn.o
targets := $(obj-vdso) vdso.so vdso.so.dbg
obj-vdso := $(addprefix $(obj)/, $(obj-vdso))
btildflags-$(CONFIG_ARM64_BTI_KERNEL) += -z force-bti
# -Bsymbolic has been added for consistency with arm, the compat vDSO and
# potential future proofing if we end up with internal calls to the exported
# routines, as x86 does (see 6f121e548f83 ("x86, vdso: Reimplement vdso.so
# preparation in build-time C")).
ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \
-Bsymbolic --eh-frame-hdr --build-id -n -T
-Bsymbolic --eh-frame-hdr --build-id -n $(btildflags-y) -T
ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18
ccflags-y += -DDISABLE_BRANCH_PROFILING

3
arch/arm64/kernel/vdso/note.S
View File

@ -12,9 +12,12 @@
#include <linux/version.h>
#include <linux/elfnote.h>
#include <linux/build-salt.h>
#include <asm/assembler.h>
ELFNOTE_START(Linux, 0, "a")
.long LINUX_VERSION_CODE
ELFNOTE_END
BUILD_SALT
emit_aarch64_feature_1_and

54
arch/arm64/kernel/vdso/sigreturn.S
View File

@ -1,7 +1,11 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Sigreturn trampoline for returning from a signal when the SA_RESTORER
* flag is not set.
* flag is not set. It serves primarily as a hall of shame for crappy
* unwinders and features an exciting but mysterious NOP instruction.
*
* It's also fragile as hell, so please think twice before changing anything
* in here.
*
* Copyright (C) 2012 ARM Limited
*
@ -9,18 +13,54 @@
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unistd.h>
.text
nop
SYM_FUNC_START(__kernel_rt_sigreturn)
/* Ensure that the mysterious NOP can be associated with a function. */
.cfi_startproc
/*
* .cfi_signal_frame causes the corresponding Frame Description Entry in the
* .eh_frame section to be annotated as a signal frame. This allows DWARF
* unwinders (e.g. libstdc++) to implement _Unwind_GetIPInfo(), which permits
* unwinding out of the signal trampoline without the need for the mysterious
* NOP.
*/
.cfi_signal_frame
.cfi_def_cfa x29, 0
.cfi_offset x29, 0 * 8
.cfi_offset x30, 1 * 8
/*
* Tell the unwinder where to locate the frame record linking back to the
* interrupted context. We don't provide unwind info for registers other
* than the frame pointer and the link register here; in practice, this
* is sufficient for unwinding in C/C++ based runtimes and the values in
* the sigcontext may have been modified by this point anyway. Debuggers
* already have baked-in strategies for attempting to unwind out of signals.
*/
.cfi_def_cfa x29, 0
.cfi_offset x29, 0 * 8
.cfi_offset x30, 1 * 8
/*
* This mysterious NOP is required for some unwinders (e.g. libc++) that
* unconditionally subtract one from the result of _Unwind_GetIP() in order to
* identify the calling function.
* Hack borrowed from arch/powerpc/kernel/vdso64/sigtramp.S.
*/
nop // Mysterious NOP
/*
* GDB relies on being able to identify the sigreturn instruction sequence to
* unwind from signal handlers. We cannot, therefore, use SYM_FUNC_START()
* here, as it will emit a BTI C instruction and break the unwinder. Thankfully,
* this function is only ever called from a RET and so omitting the landing pad
* is perfectly fine.
*/
SYM_CODE_START(__kernel_rt_sigreturn)
mov x8, #__NR_rt_sigreturn
svc #0
.cfi_endproc
SYM_FUNC_END(__kernel_rt_sigreturn)
SYM_CODE_END(__kernel_rt_sigreturn)
emit_aarch64_feature_1_and

3
arch/arm64/kernel/vdso/vdso.S
View File

@ -8,6 +8,7 @@
#include <linux/init.h>
#include <linux/linkage.h>
#include <linux/const.h>
#include <asm/assembler.h>
#include <asm/page.h>
.globl vdso_start, vdso_end
@ -19,3 +20,5 @@ vdso_start:
vdso_end:
.previous
emit_aarch64_feature_1_and

19
arch/arm64/kernel/vdso32/sigreturn.S
View File

@ -3,6 +3,9 @@
* This file provides both A32 and T32 versions, in accordance with the
* arm sigreturn code.
*
* Please read the comments in arch/arm64/kernel/vdso/sigreturn.S to
* understand some of the craziness in here.
*
* Copyright (C) 2018 ARM Limited
*/
@ -17,39 +20,39 @@
.save {r0-r15}
.pad #COMPAT_SIGFRAME_REGS_OFFSET
nop
SYM_FUNC_START(__kernel_sigreturn_arm)
SYM_CODE_START(__kernel_sigreturn_arm)
mov r7, #__NR_compat_sigreturn
svc #0
.fnend
SYM_FUNC_END(__kernel_sigreturn_arm)
SYM_CODE_END(__kernel_sigreturn_arm)
.fnstart
.save {r0-r15}
.pad #COMPAT_RT_SIGFRAME_REGS_OFFSET
nop
SYM_FUNC_START(__kernel_rt_sigreturn_arm)
SYM_CODE_START(__kernel_rt_sigreturn_arm)
mov r7, #__NR_compat_rt_sigreturn
svc #0
.fnend
SYM_FUNC_END(__kernel_rt_sigreturn_arm)
SYM_CODE_END(__kernel_rt_sigreturn_arm)
.thumb
.fnstart
.save {r0-r15}
.pad #COMPAT_SIGFRAME_REGS_OFFSET
nop
SYM_FUNC_START(__kernel_sigreturn_thumb)
SYM_CODE_START(__kernel_sigreturn_thumb)
mov r7, #__NR_compat_sigreturn
svc #0
.fnend
SYM_FUNC_END(__kernel_sigreturn_thumb)
SYM_CODE_END(__kernel_sigreturn_thumb)
.fnstart
.save {r0-r15}
.pad #COMPAT_RT_SIGFRAME_REGS_OFFSET
nop
SYM_FUNC_START(__kernel_rt_sigreturn_thumb)
SYM_CODE_START(__kernel_rt_sigreturn_thumb)
mov r7, #__NR_compat_rt_sigreturn
svc #0
.fnend
SYM_FUNC_END(__kernel_rt_sigreturn_thumb)
SYM_CODE_END(__kernel_rt_sigreturn_thumb)

5
arch/arm64/mm/dump.c
View File

@ -145,6 +145,11 @@ static const struct prot_bits pte_bits[] = {
.val = PTE_UXN,
.set = "UXN",
.clear = " ",
}, {
.mask = PTE_GP,
.val = PTE_GP,
.set = "GP",
.clear = " ",
}, {
.mask = PTE_ATTRINDX_MASK,
.val = PTE_ATTRINDX(MT_DEVICE_nGnRnE),

24
arch/arm64/mm/mmu.c
View File

@ -609,6 +609,22 @@ static int __init map_entry_trampoline(void)
core_initcall(map_entry_trampoline);
#endif
/*
* Open coded check for BTI, only for use to determine configuration
* for early mappings for before the cpufeature code has run.
*/
static bool arm64_early_this_cpu_has_bti(void)
{
u64 pfr1;
if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
return false;
pfr1 = read_sysreg_s(SYS_ID_AA64PFR1_EL1);
return cpuid_feature_extract_unsigned_field(pfr1,
ID_AA64PFR1_BT_SHIFT);
}
/*
* Create fine-grained mappings for the kernel.
*/
@ -624,6 +640,14 @@ static void __init map_kernel(pgd_t *pgdp)
*/
pgprot_t text_prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
/*
* If we have a CPU that supports BTI and a kernel built for
* BTI then mark the kernel executable text as guarded pages
* now so we don't have to rewrite the page tables later.
*/
if (arm64_early_this_cpu_has_bti())
text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP);
/*
* Only rodata will be remapped with different permissions later on,
* all other segments are allowed to use contiguous mappings.

4
arch/arm64/mm/pageattr.c
View File

@ -126,13 +126,13 @@ int set_memory_nx(unsigned long addr, int numpages)
{
return change_memory_common(addr, numpages,
__pgprot(PTE_PXN),
__pgprot(0));
__pgprot(PTE_MAYBE_GP));
}
int set_memory_x(unsigned long addr, int numpages)
{
return change_memory_common(addr, numpages,
__pgprot(0),
__pgprot(PTE_MAYBE_GP),
__pgprot(PTE_PXN));
}

8
arch/arm64/net/bpf_jit.h
View File

@ -211,4 +211,12 @@
/* Rn & imm; set condition flags */
#define A64_TST_I(sf, Rn, imm) A64_ANDS_I(sf, A64_ZR, Rn, imm)
/* HINTs */
#define A64_HINT(x) aarch64_insn_gen_hint(x)
/* BTI */
#define A64_BTI_C A64_HINT(AARCH64_INSN_HINT_BTIC)
#define A64_BTI_J A64_HINT(AARCH64_INSN_HINT_BTIJ)
#define A64_BTI_JC A64_HINT(AARCH64_INSN_HINT_BTIJC)
#endif /* _BPF_JIT_H */

12
arch/arm64/net/bpf_jit_comp.c
View File

@ -177,7 +177,11 @@ static bool is_addsub_imm(u32 imm)
#define STACK_ALIGN(sz) (((sz) + 15) & ~15)
/* Tail call offset to jump into */
#if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)
#define PROLOGUE_OFFSET 8
#else
#define PROLOGUE_OFFSET 7
#endif
static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
{
@ -214,6 +218,10 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
*
*/
/* BTI landing pad */
if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
emit(A64_BTI_C, ctx);
/* Save FP and LR registers to stay align with ARM64 AAPCS */
emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
emit(A64_MOV(1, A64_FP, A64_SP), ctx);
@ -236,6 +244,10 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
cur_offset, PROLOGUE_OFFSET);
return -1;
}
/* BTI landing pad for the tail call, done with a BR */
if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
emit(A64_BTI_J, ctx);
}
ctx->stack_size = STACK_ALIGN(prog->aux->stack_depth);

1
arch/x86/Kconfig
View File

@ -91,6 +91,7 @@ config X86
select ARCH_USE_BUILTIN_BSWAP
select ARCH_USE_QUEUED_RWLOCKS
select ARCH_USE_QUEUED_SPINLOCKS
select ARCH_USE_SYM_ANNOTATIONS
select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
select ARCH_WANT_DEFAULT_BPF_JIT if X86_64
select ARCH_WANTS_DYNAMIC_TASK_STRUCT

9
arch/x86/Kconfig.debug
View File

@ -99,15 +99,6 @@ config DEBUG_WX
If in doubt, say "Y".
config DOUBLEFAULT
default y
bool "Enable doublefault exception handler" if EXPERT && X86_32
---help---
This option allows trapping of rare doublefault exceptions that
would otherwise cause a system to silently reboot. Disabling this
option saves about 4k and might cause you much additional grey
hair.
config DEBUG_TLBFLUSH
bool "Set upper limit of TLB entries to flush one-by-one"
depends on DEBUG_KERNEL

2
arch/x86/entry/entry_32.S
View File

@ -1536,7 +1536,6 @@ SYM_CODE_START(debug)
jmp common_exception
SYM_CODE_END(debug)
#ifdef CONFIG_DOUBLEFAULT
SYM_CODE_START(double_fault)
1:
/*
@ -1576,7 +1575,6 @@ SYM_CODE_START(double_fault)
hlt
jmp 1b
SYM_CODE_END(double_fault)
#endif
/*
* NMI is doubly nasty. It can happen on the first instruction of

2
arch/x86/include/asm/doublefault.h
View File

@ -2,7 +2,7 @@
#ifndef _ASM_X86_DOUBLEFAULT_H
#define _ASM_X86_DOUBLEFAULT_H
#if defined(CONFIG_X86_32) && defined(CONFIG_DOUBLEFAULT)
#ifdef CONFIG_X86_32
extern void doublefault_init_cpu_tss(void);
#else
static inline void doublefault_init_cpu_tss(void)

2
arch/x86/include/asm/traps.h
View File

@ -69,9 +69,7 @@ dotraplinkage void do_overflow(struct pt_regs *regs, long error_code);
dotraplinkage void do_bounds(struct pt_regs *regs, long error_code);
dotraplinkage void do_invalid_op(struct pt_regs *regs, long error_code);
dotraplinkage void do_device_not_available(struct pt_regs *regs, long error_code);
#if defined(CONFIG_X86_64) || defined(CONFIG_DOUBLEFAULT)
dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2);
#endif
dotraplinkage void do_coprocessor_segment_overrun(struct pt_regs *regs, long error_code);
dotraplinkage void do_invalid_TSS(struct pt_regs *regs, long error_code);
dotraplinkage void do_segment_not_present(struct pt_regs *regs, long error_code);

4
arch/x86/kernel/Makefile
View File

@ -102,9 +102,7 @@ obj-$(CONFIG_KEXEC_FILE) += kexec-bzimage64.o
obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o
obj-y += kprobes/
obj-$(CONFIG_MODULES) += module.o
ifeq ($(CONFIG_X86_32),y)
obj-$(CONFIG_DOUBLEFAULT) += doublefault_32.o
endif
obj-$(CONFIG_X86_32) += doublefault_32.o
obj-$(CONFIG_KGDB) += kgdb.o
obj-$(CONFIG_VM86) += vm86_32.o
obj-$(CONFIG_EARLY_PRINTK) += early_printk.o

4
arch/x86/kernel/dumpstack_32.c
View File

@ -87,7 +87,6 @@ static bool in_softirq_stack(unsigned long *stack, struct stack_info *info)
static bool in_doublefault_stack(unsigned long *stack, struct stack_info *info)
{
#ifdef CONFIG_DOUBLEFAULT
struct cpu_entry_area *cea = get_cpu_entry_area(raw_smp_processor_id());
struct doublefault_stack *ss = &cea->doublefault_stack;
@ -103,9 +102,6 @@ static bool in_doublefault_stack(unsigned long *stack, struct stack_info *info)
info->next_sp = (unsigned long *)this_cpu_read(cpu_tss_rw.x86_tss.sp);
return true;
#else
return false;
#endif
}

2
arch/x86/kernel/traps.c
View File

@ -326,7 +326,6 @@ __visible void __noreturn handle_stack_overflow(const char *message,
}
#endif
#if defined(CONFIG_X86_64) || defined(CONFIG_DOUBLEFAULT)
/*
* Runs on an IST stack for x86_64 and on a special task stack for x86_32.
*
@ -450,7 +449,6 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsign
die("double fault", regs, error_code);
panic("Machine halted.");
}
#endif
dotraplinkage void do_bounds(struct pt_regs *regs, long error_code)
{

4
arch/x86/mm/cpu_entry_area.c
View File

@ -17,7 +17,7 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct exception_stacks, exception_stacks);
DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks);
#endif
#if defined(CONFIG_X86_32) && defined(CONFIG_DOUBLEFAULT)
#ifdef CONFIG_X86_32
DECLARE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack);
#endif
@ -114,12 +114,10 @@ static void __init percpu_setup_exception_stacks(unsigned int cpu)
#else
static inline void percpu_setup_exception_stacks(unsigned int cpu)
{
#ifdef CONFIG_DOUBLEFAULT
struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
cea_map_percpu_pages(&cea->doublefault_stack,
&per_cpu(doublefault_stack, cpu), 1, PAGE_KERNEL);
#endif
}
#endif

6
fs/Kconfig.binfmt
View File

@ -36,6 +36,12 @@ config COMPAT_BINFMT_ELF
config ARCH_BINFMT_ELF_STATE
bool
config ARCH_HAVE_ELF_PROT
bool
config ARCH_USE_GNU_PROPERTY
bool
config BINFMT_ELF_FDPIC
bool "Kernel support for FDPIC ELF binaries"
default y if !BINFMT_ELF

145
fs/binfmt_elf.c
View File

@ -40,12 +40,18 @@
#include <linux/sched/coredump.h>
#include <linux/sched/task_stack.h>
#include <linux/sched/cputime.h>
#include <linux/sizes.h>
#include <linux/types.h>
#include <linux/cred.h>
#include <linux/dax.h>
#include <linux/uaccess.h>
#include <asm/param.h>
#include <asm/page.h>
#ifndef ELF_COMPAT
#define ELF_COMPAT 0
#endif
#ifndef user_long_t
#define user_long_t long
#endif
@ -539,7 +545,8 @@ static inline int arch_check_elf(struct elfhdr *ehdr, bool has_interp,
#endif /* !CONFIG_ARCH_BINFMT_ELF_STATE */
static inline int make_prot(u32 p_flags)
static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state,
bool has_interp, bool is_interp)
{
int prot = 0;
@ -549,7 +556,8 @@ static inline int make_prot(u32 p_flags)
prot |= PROT_WRITE;
if (p_flags & PF_X)
prot |= PROT_EXEC;
return prot;
return arch_elf_adjust_prot(prot, arch_state, has_interp, is_interp);
}
/* This is much more generalized than the library routine read function,
@ -559,7 +567,8 @@ static inline int make_prot(u32 p_flags)
static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex,
struct file *interpreter,
unsigned long no_base, struct elf_phdr *interp_elf_phdata)
unsigned long no_base, struct elf_phdr *interp_elf_phdata,
struct arch_elf_state *arch_state)
{
struct elf_phdr *eppnt;
unsigned long load_addr = 0;
@ -591,7 +600,8 @@ static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex,
for (i = 0; i < interp_elf_ex->e_phnum; i++, eppnt++) {
if (eppnt->p_type == PT_LOAD) {
int elf_type = MAP_PRIVATE | MAP_DENYWRITE;
int elf_prot = make_prot(eppnt->p_flags);
int elf_prot = make_prot(eppnt->p_flags, arch_state,
true, true);
unsigned long vaddr = 0;
unsigned long k, map_addr;
@ -682,6 +692,111 @@ out:
* libraries. There is no binary dependent code anywhere else.
*/
static int parse_elf_property(const char *data, size_t *off, size_t datasz,
struct arch_elf_state *arch,
bool have_prev_type, u32 *prev_type)
{
size_t o, step;
const struct gnu_property *pr;
int ret;
if (*off == datasz)
return -ENOENT;
if (WARN_ON_ONCE(*off > datasz || *off % ELF_GNU_PROPERTY_ALIGN))
return -EIO;
o = *off;
datasz -= *off;
if (datasz < sizeof(*pr))
return -ENOEXEC;
pr = (const struct gnu_property *)(data + o);
o += sizeof(*pr);
datasz -= sizeof(*pr);
if (pr->pr_datasz > datasz)
return -ENOEXEC;
WARN_ON_ONCE(o % ELF_GNU_PROPERTY_ALIGN);
step = round_up(pr->pr_datasz, ELF_GNU_PROPERTY_ALIGN);
if (step > datasz)
return -ENOEXEC;
/* Properties are supposed to be unique and sorted on pr_type: */
if (have_prev_type && pr->pr_type <= *prev_type)
return -ENOEXEC;
*prev_type = pr->pr_type;
ret = arch_parse_elf_property(pr->pr_type, data + o,
pr->pr_datasz, ELF_COMPAT, arch);
if (ret)
return ret;
*off = o + step;
return 0;
}
#define NOTE_DATA_SZ SZ_1K
#define GNU_PROPERTY_TYPE_0_NAME "GNU"
#define NOTE_NAME_SZ (sizeof(GNU_PROPERTY_TYPE_0_NAME))
static int parse_elf_properties(struct file *f, const struct elf_phdr *phdr,
struct arch_elf_state *arch)
{
union {
struct elf_note nhdr;
char data[NOTE_DATA_SZ];
} note;
loff_t pos;
ssize_t n;
size_t off, datasz;
int ret;
bool have_prev_type;
u32 prev_type;
if (!IS_ENABLED(CONFIG_ARCH_USE_GNU_PROPERTY) || !phdr)
return 0;
/* load_elf_binary() shouldn't call us unless this is true... */
if (WARN_ON_ONCE(phdr->p_type != PT_GNU_PROPERTY))
return -ENOEXEC;
/* If the properties are crazy large, that's too bad (for now): */
if (phdr->p_filesz > sizeof(note))
return -ENOEXEC;
pos = phdr->p_offset;
n = kernel_read(f, &note, phdr->p_filesz, &pos);
BUILD_BUG_ON(sizeof(note) < sizeof(note.nhdr) + NOTE_NAME_SZ);
if (n < 0 || n < sizeof(note.nhdr) + NOTE_NAME_SZ)
return -EIO;
if (note.nhdr.n_type != NT_GNU_PROPERTY_TYPE_0 ||
note.nhdr.n_namesz != NOTE_NAME_SZ ||
strncmp(note.data + sizeof(note.nhdr),
GNU_PROPERTY_TYPE_0_NAME, n - sizeof(note.nhdr)))
return -ENOEXEC;
off = round_up(sizeof(note.nhdr) + NOTE_NAME_SZ,
ELF_GNU_PROPERTY_ALIGN);
if (off > n)
return -ENOEXEC;
if (note.nhdr.n_descsz > n - off)
return -ENOEXEC;
datasz = off + note.nhdr.n_descsz;
have_prev_type = false;
do {
ret = parse_elf_property(note.data, &off, datasz, arch,
have_prev_type, &prev_type);
have_prev_type = true;
} while (!ret);
return ret == -ENOENT ? 0 : ret;
}
static int load_elf_binary(struct linux_binprm *bprm)
{
struct file *interpreter = NULL; /* to shut gcc up */
@ -689,6 +804,7 @@ static int load_elf_binary(struct linux_binprm *bprm)
int load_addr_set = 0;
unsigned long error;
struct elf_phdr *elf_ppnt, *elf_phdata, *interp_elf_phdata = NULL;
struct elf_phdr *elf_property_phdata = NULL;
unsigned long elf_bss, elf_brk;
int bss_prot = 0;
int retval, i;
@ -726,6 +842,11 @@ static int load_elf_binary(struct linux_binprm *bprm)
for (i = 0; i < elf_ex->e_phnum; i++, elf_ppnt++) {
char *elf_interpreter;
if (elf_ppnt->p_type == PT_GNU_PROPERTY) {
elf_property_phdata = elf_ppnt;
continue;
}
if (elf_ppnt->p_type != PT_INTERP)
continue;
@ -819,9 +940,14 @@ out_free_interp:
goto out_free_dentry;
/* Pass PT_LOPROC..PT_HIPROC headers to arch code */
elf_property_phdata = NULL;
elf_ppnt = interp_elf_phdata;
for (i = 0; i < interp_elf_ex->e_phnum; i++, elf_ppnt++)
switch (elf_ppnt->p_type) {
case PT_GNU_PROPERTY:
elf_property_phdata = elf_ppnt;
break;
case PT_LOPROC ... PT_HIPROC:
retval = arch_elf_pt_proc(interp_elf_ex,
elf_ppnt, interpreter,
@ -832,6 +958,11 @@ out_free_interp:
}
}
retval = parse_elf_properties(interpreter ?: bprm->file,
elf_property_phdata, &arch_state);
if (retval)
goto out_free_dentry;
/*
* Allow arch code to reject the ELF at this point, whilst it's
* still possible to return an error to the code that invoked
@ -913,7 +1044,8 @@ out_free_interp:
}
}
elf_prot = make_prot(elf_ppnt->p_flags);
elf_prot = make_prot(elf_ppnt->p_flags, &arch_state,
!!interpreter, false);
elf_flags = MAP_PRIVATE | MAP_DENYWRITE | MAP_EXECUTABLE;
@ -1056,7 +1188,8 @@ out_free_interp:
if (interpreter) {
elf_entry = load_elf_interp(interp_elf_ex,
interpreter,
load_bias, interp_elf_phdata);
load_bias, interp_elf_phdata,
&arch_state);
if (!IS_ERR((void *)elf_entry)) {
/*
* load_elf_interp() returns relocation

4
fs/compat_binfmt_elf.c
View File

@ -17,6 +17,8 @@
#include <linux/elfcore-compat.h>
#include <linux/time.h>
#define ELF_COMPAT 1
/*
* Rename the basic ELF layout types to refer to the 32-bit class of files.
*/
@ -28,11 +30,13 @@
#undef elf_shdr
#undef elf_note
#undef elf_addr_t
#undef ELF_GNU_PROPERTY_ALIGN
#define elfhdr elf32_hdr
#define elf_phdr elf32_phdr
#define elf_shdr elf32_shdr
#define elf_note elf32_note
#define elf_addr_t Elf32_Addr
#define ELF_GNU_PROPERTY_ALIGN ELF32_GNU_PROPERTY_ALIGN
/*
* Some data types as stored in coredump.

3
fs/proc/task_mmu.c
View File

@ -638,6 +638,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
[ilog2(VM_ARCH_1)] = "ar",
[ilog2(VM_WIPEONFORK)] = "wf",
[ilog2(VM_DONTDUMP)] = "dd",
#ifdef CONFIG_ARM64_BTI
[ilog2(VM_ARM64_BTI)] = "bt",
#endif
#ifdef CONFIG_MEM_SOFT_DIRTY
[ilog2(VM_SOFTDIRTY)] = "sd",
#endif

43
include/linux/elf.h
View File

@ -2,6 +2,7 @@
#ifndef _LINUX_ELF_H
#define _LINUX_ELF_H
#include <linux/types.h>
#include <asm/elf.h>
#include <uapi/linux/elf.h>
@ -21,6 +22,9 @@
SET_PERSONALITY(ex)
#endif
#define ELF32_GNU_PROPERTY_ALIGN 4
#define ELF64_GNU_PROPERTY_ALIGN 8
#if ELF_CLASS == ELFCLASS32
extern Elf32_Dyn _DYNAMIC [];
@ -31,6 +35,7 @@ extern Elf32_Dyn _DYNAMIC [];
#define elf_addr_t Elf32_Off
#define Elf_Half Elf32_Half
#define Elf_Word Elf32_Word
#define ELF_GNU_PROPERTY_ALIGN ELF32_GNU_PROPERTY_ALIGN
#else
@ -42,6 +47,7 @@ extern Elf64_Dyn _DYNAMIC [];
#define elf_addr_t Elf64_Off
#define Elf_Half Elf64_Half
#define Elf_Word Elf64_Word
#define ELF_GNU_PROPERTY_ALIGN ELF64_GNU_PROPERTY_ALIGN
#endif
@ -56,4 +62,41 @@ static inline int elf_coredump_extra_notes_write(struct coredump_params *cprm) {
extern int elf_coredump_extra_notes_size(void);
extern int elf_coredump_extra_notes_write(struct coredump_params *cprm);
#endif
/*
* NT_GNU_PROPERTY_TYPE_0 header:
* Keep this internal until/unless there is an agreed UAPI definition.
* pr_type values (GNU_PROPERTY_*) are public and defined in the UAPI header.
*/
struct gnu_property {
u32 pr_type;
u32 pr_datasz;
};
struct arch_elf_state;
#ifndef CONFIG_ARCH_USE_GNU_PROPERTY
static inline int arch_parse_elf_property(u32 type, const void *data,
size_t datasz, bool compat,
struct arch_elf_state *arch)
{
return 0;
}
#else
extern int arch_parse_elf_property(u32 type, const void *data, size_t datasz,
bool compat, struct arch_elf_state *arch);
#endif
#ifdef CONFIG_ARCH_HAVE_ELF_PROT
int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state,
bool has_interp, bool is_interp);
#else
static inline int arch_elf_adjust_prot(int prot,
const struct arch_elf_state *state,
bool has_interp, bool is_interp)
{
return prot;
}
#endif
#endif /* _LINUX_ELF_H */

8
include/linux/linkage.h
View File

@ -105,7 +105,7 @@
/* === DEPRECATED annotations === */
#ifndef CONFIG_X86
#ifndef CONFIG_ARCH_USE_SYM_ANNOTATIONS
#ifndef GLOBAL
/* deprecated, use SYM_DATA*, SYM_ENTRY, or similar */
#define GLOBAL(name) \
@ -118,10 +118,10 @@
#define ENTRY(name) \
SYM_FUNC_START(name)
#endif
#endif /* CONFIG_X86 */
#endif /* CONFIG_ARCH_USE_SYM_ANNOTATIONS */
#endif /* LINKER_SCRIPT */
#ifndef CONFIG_X86
#ifndef CONFIG_ARCH_USE_SYM_ANNOTATIONS
#ifndef WEAK
/* deprecated, use SYM_FUNC_START_WEAK* */
#define WEAK(name) \
@ -143,7 +143,7 @@
#define ENDPROC(name) \
SYM_FUNC_END(name)
#endif
#endif /* CONFIG_X86 */
#endif /* CONFIG_ARCH_USE_SYM_ANNOTATIONS */
/* === generic annotations === */

3
include/linux/mm.h
View File

@ -325,6 +325,9 @@ extern unsigned int kobjsize(const void *objp);
#elif defined(CONFIG_SPARC64)
# define VM_SPARC_ADI VM_ARCH_1 /* Uses ADI tag for access control */
# define VM_ARCH_CLEAR VM_SPARC_ADI
#elif defined(CONFIG_ARM64)
# define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */
# define VM_ARCH_CLEAR VM_ARM64_BTI
#elif !defined(CONFIG_MMU)
# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
#endif

11
include/uapi/linux/elf.h
View File

@ -36,6 +36,7 @@ typedef __s64 Elf64_Sxword;
#define PT_LOPROC 0x70000000
#define PT_HIPROC 0x7fffffff
#define PT_GNU_EH_FRAME 0x6474e550
#define PT_GNU_PROPERTY 0x6474e553
#define PT_GNU_STACK (PT_LOOS + 0x474e551)
@ -367,6 +368,7 @@ typedef struct elf64_shdr {
* Notes used in ET_CORE. Architectures export some of the arch register sets
* using the corresponding note types via the PTRACE_GETREGSET and
* PTRACE_SETREGSET requests.
* The note name for all these is "LINUX".
*/
#define NT_PRSTATUS 1
#define NT_PRFPREG 2
@ -429,6 +431,9 @@ typedef struct elf64_shdr {
#define NT_MIPS_FP_MODE 0x801 /* MIPS floating-point mode */
#define NT_MIPS_MSA 0x802 /* MIPS SIMD registers */
/* Note types with note name "GNU" */
#define NT_GNU_PROPERTY_TYPE_0 5
/* Note header in a PT_NOTE section */
typedef struct elf32_note {
Elf32_Word n_namesz; /* Name size */
@ -443,4 +448,10 @@ typedef struct elf64_note {
Elf64_Word n_type; /* Content type */
} Elf64_Nhdr;
/* .note.gnu.property types for EM_AARCH64: */
#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
/* Bits for GNU_PROPERTY_AARCH64_FEATURE_1_BTI */
#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0)
#endif /* _UAPI_LINUX_ELF_H */

3
lib/Kconfig
View File

@ -80,6 +80,9 @@ config ARCH_USE_CMPXCHG_LOCKREF
config ARCH_HAS_FAST_MULTIPLIER
bool
config ARCH_USE_SYM_ANNOTATIONS
bool
config INDIRECT_PIO
bool "Access I/O in non-MMIO mode"
depends on ARM64

1
tools/testing/selftests/wireguard/qemu/debug.config
View File

@ -58,7 +58,6 @@ CONFIG_RCU_EQS_DEBUG=y
CONFIG_USER_STACKTRACE_SUPPORT=y
CONFIG_DEBUG_SG=y
CONFIG_DEBUG_NOTIFIERS=y
CONFIG_DOUBLEFAULT=y
CONFIG_X86_DEBUG_FPU=y
CONFIG_DEBUG_SECTION_MISMATCH=y
CONFIG_DEBUG_PAGEALLOC=y