46808a4cb8
Everywhere we use kvm_for_each_vpcu(), we use an int as the vcpu index. Unfortunately, we're about to move rework the iterator, which requires this to be upgrade to an unsigned long. Let's bite the bullet and repaint all of it in one go. Signed-off-by: Marc Zyngier <maz@kernel.org> Message-Id: <20211116160403.4074052-7-maz@kernel.org> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
121 lines
3.0 KiB
C
121 lines
3.0 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright (C) 2019 Western Digital Corporation or its affiliates.
|
|
*
|
|
* Authors:
|
|
* Anup Patel <anup.patel@wdc.com>
|
|
*/
|
|
|
|
#include <linux/bitops.h>
|
|
#include <linux/cpumask.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/err.h>
|
|
#include <linux/module.h>
|
|
#include <linux/kvm_host.h>
|
|
#include <asm/csr.h>
|
|
#include <asm/sbi.h>
|
|
|
|
static unsigned long vmid_version = 1;
|
|
static unsigned long vmid_next;
|
|
static unsigned long vmid_bits;
|
|
static DEFINE_SPINLOCK(vmid_lock);
|
|
|
|
void kvm_riscv_stage2_vmid_detect(void)
|
|
{
|
|
unsigned long old;
|
|
|
|
/* Figure-out number of VMID bits in HW */
|
|
old = csr_read(CSR_HGATP);
|
|
csr_write(CSR_HGATP, old | HGATP_VMID_MASK);
|
|
vmid_bits = csr_read(CSR_HGATP);
|
|
vmid_bits = (vmid_bits & HGATP_VMID_MASK) >> HGATP_VMID_SHIFT;
|
|
vmid_bits = fls_long(vmid_bits);
|
|
csr_write(CSR_HGATP, old);
|
|
|
|
/* We polluted local TLB so flush all guest TLB */
|
|
__kvm_riscv_hfence_gvma_all();
|
|
|
|
/* We don't use VMID bits if they are not sufficient */
|
|
if ((1UL << vmid_bits) < num_possible_cpus())
|
|
vmid_bits = 0;
|
|
}
|
|
|
|
unsigned long kvm_riscv_stage2_vmid_bits(void)
|
|
{
|
|
return vmid_bits;
|
|
}
|
|
|
|
int kvm_riscv_stage2_vmid_init(struct kvm *kvm)
|
|
{
|
|
/* Mark the initial VMID and VMID version invalid */
|
|
kvm->arch.vmid.vmid_version = 0;
|
|
kvm->arch.vmid.vmid = 0;
|
|
|
|
return 0;
|
|
}
|
|
|
|
bool kvm_riscv_stage2_vmid_ver_changed(struct kvm_vmid *vmid)
|
|
{
|
|
if (!vmid_bits)
|
|
return false;
|
|
|
|
return unlikely(READ_ONCE(vmid->vmid_version) !=
|
|
READ_ONCE(vmid_version));
|
|
}
|
|
|
|
void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu)
|
|
{
|
|
unsigned long i;
|
|
struct kvm_vcpu *v;
|
|
struct cpumask hmask;
|
|
struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid;
|
|
|
|
if (!kvm_riscv_stage2_vmid_ver_changed(vmid))
|
|
return;
|
|
|
|
spin_lock(&vmid_lock);
|
|
|
|
/*
|
|
* We need to re-check the vmid_version here to ensure that if
|
|
* another vcpu already allocated a valid vmid for this vm.
|
|
*/
|
|
if (!kvm_riscv_stage2_vmid_ver_changed(vmid)) {
|
|
spin_unlock(&vmid_lock);
|
|
return;
|
|
}
|
|
|
|
/* First user of a new VMID version? */
|
|
if (unlikely(vmid_next == 0)) {
|
|
WRITE_ONCE(vmid_version, READ_ONCE(vmid_version) + 1);
|
|
vmid_next = 1;
|
|
|
|
/*
|
|
* We ran out of VMIDs so we increment vmid_version and
|
|
* start assigning VMIDs from 1.
|
|
*
|
|
* This also means existing VMIDs assignement to all Guest
|
|
* instances is invalid and we have force VMID re-assignement
|
|
* for all Guest instances. The Guest instances that were not
|
|
* running will automatically pick-up new VMIDs because will
|
|
* call kvm_riscv_stage2_vmid_update() whenever they enter
|
|
* in-kernel run loop. For Guest instances that are already
|
|
* running, we force VM exits on all host CPUs using IPI and
|
|
* flush all Guest TLBs.
|
|
*/
|
|
riscv_cpuid_to_hartid_mask(cpu_online_mask, &hmask);
|
|
sbi_remote_hfence_gvma(cpumask_bits(&hmask), 0, 0);
|
|
}
|
|
|
|
vmid->vmid = vmid_next;
|
|
vmid_next++;
|
|
vmid_next &= (1 << vmid_bits) - 1;
|
|
|
|
WRITE_ONCE(vmid->vmid_version, READ_ONCE(vmid_version));
|
|
|
|
spin_unlock(&vmid_lock);
|
|
|
|
/* Request stage2 page table update for all VCPUs */
|
|
kvm_for_each_vcpu(i, v, vcpu->kvm)
|
|
kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
|
|
}
|