mirror of
https://github.com/torvalds/linux.git
synced 2024-11-15 08:31:55 +00:00
dcf89d1111
Some bti instructions were missing from commitb53d4a2723("KVM: arm64: Use BTI for nvhe") 1) kvm_host_psci_cpu_entry kvm_host_psci_cpu_entry is called from __kvm_hyp_init_cpu through "br" instruction as __kvm_hyp_init_cpu resides in idmap section while kvm_host_psci_cpu_entry is in hyp .text so the offset is larger than 128MB range covered by "b". Which means that this function should start with "bti j" instruction. LLVM which is the only compiler supporting BTI for Linux, adds "bti j" for jump tables or by when taking the address of the block [1]. Same behaviour is observed with GCC. As kvm_host_psci_cpu_entry is a C function, this must be done in assembly. Another solution is to use X16/X17 with "br", as according to ARM ARM DDI0487I.a RLJHCL/IGMGRS, PACIASP has an implicit branch target identification instruction that is compatible with PSTATE.BTYPE 0b01 which includes "br X16/X17" And the kvm_host_psci_cpu_entry has PACIASP as it is an external function. Although, using explicit "bti" makes it more clear than relying on which register is used. A third solution is to clear SCTLR_EL2.BT, which would make PACIASP compatible PSTATE.BTYPE 0b11 ("br" to other registers). However this deviates from the kernel behaviour (in bti_enable()). 2) Spectre vector table "br" instructions are generated at runtime for the vector table (__bp_harden_hyp_vecs). These branches would land on vectors in __kvm_hyp_vector at offset 8. As all the macros are defined with valid_vect/invalid_vect, it is sufficient to add "bti j" at the correct offset. [1] https://reviews.llvm.org/D52867 Fixes:b53d4a2723("KVM: arm64: Use BTI for nvhe") Signed-off-by: Mostafa Saleh <smostafa@google.com> Reported-by: Sudeep Holla <sudeep.holla@arm.com> Acked-by: Marc Zyngier <maz@kernel.org> Tested-by: Sudeep Holla <sudeep.holla@arm.com> Link: https://lore.kernel.org/r/20230706152240.685684-1-smostafa@google.com Signed-off-by: Oliver Upton <oliver.upton@linux.dev>
265 lines
5.9 KiB
ArmAsm
265 lines
5.9 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2015-2018 - ARM Ltd
|
|
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
|
*/
|
|
|
|
#include <linux/arm-smccc.h>
|
|
#include <linux/linkage.h>
|
|
|
|
#include <asm/alternative.h>
|
|
#include <asm/assembler.h>
|
|
#include <asm/cpufeature.h>
|
|
#include <asm/kvm_arm.h>
|
|
#include <asm/kvm_asm.h>
|
|
#include <asm/mmu.h>
|
|
#include <asm/spectre.h>
|
|
|
|
.macro save_caller_saved_regs_vect
|
|
/* x0 and x1 were saved in the vector entry */
|
|
stp x2, x3, [sp, #-16]!
|
|
stp x4, x5, [sp, #-16]!
|
|
stp x6, x7, [sp, #-16]!
|
|
stp x8, x9, [sp, #-16]!
|
|
stp x10, x11, [sp, #-16]!
|
|
stp x12, x13, [sp, #-16]!
|
|
stp x14, x15, [sp, #-16]!
|
|
stp x16, x17, [sp, #-16]!
|
|
.endm
|
|
|
|
.macro restore_caller_saved_regs_vect
|
|
ldp x16, x17, [sp], #16
|
|
ldp x14, x15, [sp], #16
|
|
ldp x12, x13, [sp], #16
|
|
ldp x10, x11, [sp], #16
|
|
ldp x8, x9, [sp], #16
|
|
ldp x6, x7, [sp], #16
|
|
ldp x4, x5, [sp], #16
|
|
ldp x2, x3, [sp], #16
|
|
ldp x0, x1, [sp], #16
|
|
.endm
|
|
|
|
.text
|
|
|
|
el1_sync: // Guest trapped into EL2
|
|
|
|
mrs x0, esr_el2
|
|
ubfx x0, x0, #ESR_ELx_EC_SHIFT, #ESR_ELx_EC_WIDTH
|
|
cmp x0, #ESR_ELx_EC_HVC64
|
|
ccmp x0, #ESR_ELx_EC_HVC32, #4, ne
|
|
b.ne el1_trap
|
|
|
|
/*
|
|
* Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1.
|
|
* The workaround has already been applied on the host,
|
|
* so let's quickly get back to the guest. We don't bother
|
|
* restoring x1, as it can be clobbered anyway.
|
|
*/
|
|
ldr x1, [sp] // Guest's x0
|
|
eor w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1
|
|
cbz w1, wa_epilogue
|
|
|
|
/* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
|
|
eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
|
|
ARM_SMCCC_ARCH_WORKAROUND_2)
|
|
cbz w1, wa_epilogue
|
|
|
|
eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \
|
|
ARM_SMCCC_ARCH_WORKAROUND_3)
|
|
cbnz w1, el1_trap
|
|
|
|
wa_epilogue:
|
|
mov x0, xzr
|
|
add sp, sp, #16
|
|
eret
|
|
sb
|
|
|
|
el1_trap:
|
|
get_vcpu_ptr x1, x0
|
|
mov x0, #ARM_EXCEPTION_TRAP
|
|
b __guest_exit
|
|
|
|
el1_irq:
|
|
el1_fiq:
|
|
get_vcpu_ptr x1, x0
|
|
mov x0, #ARM_EXCEPTION_IRQ
|
|
b __guest_exit
|
|
|
|
el1_error:
|
|
get_vcpu_ptr x1, x0
|
|
mov x0, #ARM_EXCEPTION_EL1_SERROR
|
|
b __guest_exit
|
|
|
|
el2_sync:
|
|
/* Check for illegal exception return */
|
|
mrs x0, spsr_el2
|
|
tbnz x0, #20, 1f
|
|
|
|
save_caller_saved_regs_vect
|
|
stp x29, x30, [sp, #-16]!
|
|
bl kvm_unexpected_el2_exception
|
|
ldp x29, x30, [sp], #16
|
|
restore_caller_saved_regs_vect
|
|
|
|
eret
|
|
|
|
1:
|
|
/* Let's attempt a recovery from the illegal exception return */
|
|
get_vcpu_ptr x1, x0
|
|
mov x0, #ARM_EXCEPTION_IL
|
|
b __guest_exit
|
|
|
|
|
|
el2_error:
|
|
save_caller_saved_regs_vect
|
|
stp x29, x30, [sp, #-16]!
|
|
|
|
bl kvm_unexpected_el2_exception
|
|
|
|
ldp x29, x30, [sp], #16
|
|
restore_caller_saved_regs_vect
|
|
|
|
eret
|
|
sb
|
|
|
|
.macro invalid_vector label, target = __guest_exit_panic
|
|
.align 2
|
|
SYM_CODE_START_LOCAL(\label)
|
|
b \target
|
|
SYM_CODE_END(\label)
|
|
.endm
|
|
|
|
/* None of these should ever happen */
|
|
invalid_vector el2t_sync_invalid
|
|
invalid_vector el2t_irq_invalid
|
|
invalid_vector el2t_fiq_invalid
|
|
invalid_vector el2t_error_invalid
|
|
invalid_vector el2h_irq_invalid
|
|
invalid_vector el2h_fiq_invalid
|
|
|
|
.ltorg
|
|
|
|
.align 11
|
|
|
|
.macro check_preamble_length start, end
|
|
/* kvm_patch_vector_branch() generates code that jumps over the preamble. */
|
|
.if ((\end-\start) != KVM_VECTOR_PREAMBLE)
|
|
.error "KVM vector preamble length mismatch"
|
|
.endif
|
|
.endm
|
|
|
|
.macro valid_vect target
|
|
.align 7
|
|
661:
|
|
esb
|
|
stp x0, x1, [sp, #-16]!
|
|
662:
|
|
/*
|
|
* spectre vectors __bp_harden_hyp_vecs generate br instructions at runtime
|
|
* that jump at offset 8 at __kvm_hyp_vector.
|
|
* As hyp .text is guarded section, it needs bti j.
|
|
*/
|
|
bti j
|
|
b \target
|
|
|
|
check_preamble_length 661b, 662b
|
|
.endm
|
|
|
|
.macro invalid_vect target
|
|
.align 7
|
|
661:
|
|
nop
|
|
stp x0, x1, [sp, #-16]!
|
|
662:
|
|
/* Check valid_vect */
|
|
bti j
|
|
b \target
|
|
|
|
check_preamble_length 661b, 662b
|
|
.endm
|
|
|
|
SYM_CODE_START(__kvm_hyp_vector)
|
|
invalid_vect el2t_sync_invalid // Synchronous EL2t
|
|
invalid_vect el2t_irq_invalid // IRQ EL2t
|
|
invalid_vect el2t_fiq_invalid // FIQ EL2t
|
|
invalid_vect el2t_error_invalid // Error EL2t
|
|
|
|
valid_vect el2_sync // Synchronous EL2h
|
|
invalid_vect el2h_irq_invalid // IRQ EL2h
|
|
invalid_vect el2h_fiq_invalid // FIQ EL2h
|
|
valid_vect el2_error // Error EL2h
|
|
|
|
valid_vect el1_sync // Synchronous 64-bit EL1
|
|
valid_vect el1_irq // IRQ 64-bit EL1
|
|
valid_vect el1_fiq // FIQ 64-bit EL1
|
|
valid_vect el1_error // Error 64-bit EL1
|
|
|
|
valid_vect el1_sync // Synchronous 32-bit EL1
|
|
valid_vect el1_irq // IRQ 32-bit EL1
|
|
valid_vect el1_fiq // FIQ 32-bit EL1
|
|
valid_vect el1_error // Error 32-bit EL1
|
|
SYM_CODE_END(__kvm_hyp_vector)
|
|
|
|
.macro spectrev2_smccc_wa1_smc
|
|
sub sp, sp, #(8 * 4)
|
|
stp x2, x3, [sp, #(8 * 0)]
|
|
stp x0, x1, [sp, #(8 * 2)]
|
|
alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_wa3
|
|
/* Patched to mov WA3 when supported */
|
|
mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1
|
|
alternative_cb_end
|
|
smc #0
|
|
ldp x2, x3, [sp, #(8 * 0)]
|
|
add sp, sp, #(8 * 2)
|
|
.endm
|
|
|
|
.macro hyp_ventry indirect, spectrev2
|
|
.align 7
|
|
1: esb
|
|
.if \spectrev2 != 0
|
|
spectrev2_smccc_wa1_smc
|
|
.else
|
|
stp x0, x1, [sp, #-16]!
|
|
mitigate_spectre_bhb_loop x0
|
|
mitigate_spectre_bhb_clear_insn
|
|
.endif
|
|
.if \indirect != 0
|
|
alternative_cb ARM64_ALWAYS_SYSTEM, kvm_patch_vector_branch
|
|
/*
|
|
* For ARM64_SPECTRE_V3A configurations, these NOPs get replaced with:
|
|
*
|
|
* movz x0, #(addr & 0xffff)
|
|
* movk x0, #((addr >> 16) & 0xffff), lsl #16
|
|
* movk x0, #((addr >> 32) & 0xffff), lsl #32
|
|
* br x0
|
|
*
|
|
* Where:
|
|
* addr = kern_hyp_va(__kvm_hyp_vector) + vector-offset + KVM_VECTOR_PREAMBLE.
|
|
* See kvm_patch_vector_branch for details.
|
|
*/
|
|
nop
|
|
nop
|
|
nop
|
|
nop
|
|
alternative_cb_end
|
|
.endif
|
|
b __kvm_hyp_vector + (1b - 0b + KVM_VECTOR_PREAMBLE)
|
|
.endm
|
|
|
|
.macro generate_vectors indirect, spectrev2
|
|
0:
|
|
.rept 16
|
|
hyp_ventry \indirect, \spectrev2
|
|
.endr
|
|
.org 0b + SZ_2K // Safety measure
|
|
.endm
|
|
|
|
.align 11
|
|
SYM_CODE_START(__bp_harden_hyp_vecs)
|
|
generate_vectors indirect = 0, spectrev2 = 1 // HYP_VECTOR_SPECTRE_DIRECT
|
|
generate_vectors indirect = 1, spectrev2 = 0 // HYP_VECTOR_INDIRECT
|
|
generate_vectors indirect = 1, spectrev2 = 1 // HYP_VECTOR_SPECTRE_INDIRECT
|
|
1: .org __bp_harden_hyp_vecs + __BP_HARDEN_HYP_VECS_SZ
|
|
.org 1b
|
|
SYM_CODE_END(__bp_harden_hyp_vecs)
|