mirror of
https://github.com/torvalds/linux.git
synced 2024-12-22 10:56:40 +00:00
Merge branches 'arm/smmu', 'mediatek', 's390', 'ti/omap', 'riscv' and 'core' into next
This commit is contained in:
commit
ae3325f752
@ -36,10 +36,12 @@ properties:
|
||||
items:
|
||||
- enum:
|
||||
- qcom,qcm2290-smmu-500
|
||||
- qcom,qcs615-smmu-500
|
||||
- qcom,qcs8300-smmu-500
|
||||
- qcom,qdu1000-smmu-500
|
||||
- qcom,sa8255p-smmu-500
|
||||
- qcom,sa8775p-smmu-500
|
||||
- qcom,sar2130p-smmu-500
|
||||
- qcom,sc7180-smmu-500
|
||||
- qcom,sc7280-smmu-500
|
||||
- qcom,sc8180x-smmu-500
|
||||
@ -88,6 +90,7 @@ properties:
|
||||
- qcom,qcm2290-smmu-500
|
||||
- qcom,sa8255p-smmu-500
|
||||
- qcom,sa8775p-smmu-500
|
||||
- qcom,sar2130p-smmu-500
|
||||
- qcom,sc7280-smmu-500
|
||||
- qcom,sc8180x-smmu-500
|
||||
- qcom,sc8280xp-smmu-500
|
||||
@ -524,6 +527,7 @@ allOf:
|
||||
compatible:
|
||||
items:
|
||||
- enum:
|
||||
- qcom,sar2130p-smmu-500
|
||||
- qcom,sm8550-smmu-500
|
||||
- qcom,sm8650-smmu-500
|
||||
- qcom,x1e80100-smmu-500
|
||||
@ -555,6 +559,7 @@ allOf:
|
||||
- cavium,smmu-v2
|
||||
- marvell,ap806-smmu-500
|
||||
- nvidia,smmu-500
|
||||
- qcom,qcs615-smmu-500
|
||||
- qcom,qcs8300-smmu-500
|
||||
- qcom,qdu1000-smmu-500
|
||||
- qcom,sa8255p-smmu-500
|
||||
|
147
Documentation/devicetree/bindings/iommu/riscv,iommu.yaml
Normal file
147
Documentation/devicetree/bindings/iommu/riscv,iommu.yaml
Normal file
@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
|
||||
%YAML 1.2
|
||||
---
|
||||
$id: http://devicetree.org/schemas/iommu/riscv,iommu.yaml#
|
||||
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||
|
||||
title: RISC-V IOMMU Architecture Implementation
|
||||
|
||||
maintainers:
|
||||
- Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||
|
||||
description: |
|
||||
The RISC-V IOMMU provides memory address translation and isolation for
|
||||
input and output devices, supporting per-device translation context,
|
||||
shared process address spaces including the ATS and PRI components of
|
||||
the PCIe specification, two stage address translation and MSI remapping.
|
||||
It supports identical translation table format to the RISC-V address
|
||||
translation tables with page level access and protection attributes.
|
||||
Hardware uses in-memory command and fault reporting queues with wired
|
||||
interrupt or MSI notifications.
|
||||
|
||||
Visit https://github.com/riscv-non-isa/riscv-iommu for more details.
|
||||
|
||||
For information on assigning RISC-V IOMMU to its peripheral devices,
|
||||
see generic IOMMU bindings.
|
||||
|
||||
properties:
|
||||
# For PCIe IOMMU hardware compatible property should contain the vendor
|
||||
# and device ID according to the PCI Bus Binding specification.
|
||||
# Since PCI provides built-in identification methods, compatible is not
|
||||
# actually required. For non-PCIe hardware implementations 'riscv,iommu'
|
||||
# should be specified along with 'reg' property providing MMIO location.
|
||||
compatible:
|
||||
oneOf:
|
||||
- items:
|
||||
- enum:
|
||||
- qemu,riscv-iommu
|
||||
- const: riscv,iommu
|
||||
- items:
|
||||
- enum:
|
||||
- pci1efd,edf1
|
||||
- const: riscv,pci-iommu
|
||||
|
||||
reg:
|
||||
maxItems: 1
|
||||
description:
|
||||
For non-PCI devices this represents base address and size of for the
|
||||
IOMMU memory mapped registers interface.
|
||||
For PCI IOMMU hardware implementation this should represent an address
|
||||
of the IOMMU, as defined in the PCI Bus Binding reference.
|
||||
|
||||
'#iommu-cells':
|
||||
const: 1
|
||||
description:
|
||||
The single cell describes the requester id emitted by a master to the
|
||||
IOMMU.
|
||||
|
||||
interrupts:
|
||||
minItems: 1
|
||||
maxItems: 4
|
||||
description:
|
||||
Wired interrupt vectors available for RISC-V IOMMU to notify the
|
||||
RISC-V HARTS. The cause to interrupt vector is software defined
|
||||
using IVEC IOMMU register.
|
||||
|
||||
msi-parent: true
|
||||
|
||||
power-domains:
|
||||
maxItems: 1
|
||||
|
||||
required:
|
||||
- compatible
|
||||
- reg
|
||||
- '#iommu-cells'
|
||||
|
||||
additionalProperties: false
|
||||
|
||||
examples:
|
||||
- |+
|
||||
/* Example 1 (IOMMU device with wired interrupts) */
|
||||
#include <dt-bindings/interrupt-controller/irq.h>
|
||||
|
||||
iommu1: iommu@1bccd000 {
|
||||
compatible = "qemu,riscv-iommu", "riscv,iommu";
|
||||
reg = <0x1bccd000 0x1000>;
|
||||
interrupt-parent = <&aplic_smode>;
|
||||
interrupts = <32 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<33 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<34 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<35 IRQ_TYPE_LEVEL_HIGH>;
|
||||
#iommu-cells = <1>;
|
||||
};
|
||||
|
||||
/* Device with two IOMMU device IDs, 0 and 7 */
|
||||
master1 {
|
||||
iommus = <&iommu1 0>, <&iommu1 7>;
|
||||
};
|
||||
|
||||
- |+
|
||||
/* Example 2 (IOMMU device with shared wired interrupt) */
|
||||
#include <dt-bindings/interrupt-controller/irq.h>
|
||||
|
||||
iommu2: iommu@1bccd000 {
|
||||
compatible = "qemu,riscv-iommu", "riscv,iommu";
|
||||
reg = <0x1bccd000 0x1000>;
|
||||
interrupt-parent = <&aplic_smode>;
|
||||
interrupts = <32 IRQ_TYPE_LEVEL_HIGH>;
|
||||
#iommu-cells = <1>;
|
||||
};
|
||||
|
||||
- |+
|
||||
/* Example 3 (IOMMU device with MSIs) */
|
||||
iommu3: iommu@1bcdd000 {
|
||||
compatible = "qemu,riscv-iommu", "riscv,iommu";
|
||||
reg = <0x1bccd000 0x1000>;
|
||||
msi-parent = <&imsics_smode>;
|
||||
#iommu-cells = <1>;
|
||||
};
|
||||
|
||||
- |+
|
||||
/* Example 4 (IOMMU PCIe device with MSIs) */
|
||||
bus {
|
||||
#address-cells = <2>;
|
||||
#size-cells = <2>;
|
||||
|
||||
pcie@30000000 {
|
||||
device_type = "pci";
|
||||
#address-cells = <3>;
|
||||
#size-cells = <2>;
|
||||
reg = <0x0 0x30000000 0x0 0x1000000>;
|
||||
ranges = <0x02000000 0x0 0x41000000 0x0 0x41000000 0x0 0x0f000000>;
|
||||
|
||||
/*
|
||||
* The IOMMU manages all functions in this PCI domain except
|
||||
* itself. Omit BDF 00:01.0.
|
||||
*/
|
||||
iommu-map = <0x0 &iommu0 0x0 0x8>,
|
||||
<0x9 &iommu0 0x9 0xfff7>;
|
||||
|
||||
/* The IOMMU programming interface uses slot 00:01.0 */
|
||||
iommu0: iommu@1,0 {
|
||||
compatible = "pci1efd,edf1", "riscv,pci-iommu";
|
||||
reg = <0x800 0 0 0 0>;
|
||||
#iommu-cells = <1>;
|
||||
};
|
||||
};
|
||||
};
|
@ -19810,6 +19810,15 @@ F: arch/riscv/
|
||||
N: riscv
|
||||
K: riscv
|
||||
|
||||
RISC-V IOMMU
|
||||
M: Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||
L: iommu@lists.linux.dev
|
||||
L: linux-riscv@lists.infradead.org
|
||||
S: Maintained
|
||||
T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git
|
||||
F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml
|
||||
F: drivers/iommu/riscv/
|
||||
|
||||
RISC-V MICROCHIP FPGA SUPPORT
|
||||
M: Conor Dooley <conor.dooley@microchip.com>
|
||||
M: Daire McNamara <daire.mcnamara@microchip.com>
|
||||
|
@ -96,7 +96,6 @@ struct zpci_bar_struct {
|
||||
u8 size; /* order 2 exponent */
|
||||
};
|
||||
|
||||
struct s390_domain;
|
||||
struct kvm_zdev;
|
||||
|
||||
#define ZPCI_FUNCTIONS_PER_BUS 256
|
||||
@ -181,9 +180,10 @@ struct zpci_dev {
|
||||
struct dentry *debugfs_dev;
|
||||
|
||||
/* IOMMU and passthrough */
|
||||
struct s390_domain *s390_domain; /* s390 IOMMU domain data */
|
||||
struct iommu_domain *s390_domain; /* attached IOMMU domain */
|
||||
struct kvm_zdev *kzdev;
|
||||
struct mutex kzdev_lock;
|
||||
spinlock_t dom_lock; /* protect s390_domain change */
|
||||
};
|
||||
|
||||
static inline bool zdev_enabled(struct zpci_dev *zdev)
|
||||
|
@ -160,6 +160,7 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev)
|
||||
u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
|
||||
struct zpci_iommu_ctrs *ctrs;
|
||||
struct zpci_fib fib = {0};
|
||||
unsigned long flags;
|
||||
u8 cc, status;
|
||||
|
||||
if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
|
||||
@ -171,6 +172,7 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev)
|
||||
WARN_ON((u64) zdev->fmb & 0xf);
|
||||
|
||||
/* reset software counters */
|
||||
spin_lock_irqsave(&zdev->dom_lock, flags);
|
||||
ctrs = zpci_get_iommu_ctrs(zdev);
|
||||
if (ctrs) {
|
||||
atomic64_set(&ctrs->mapped_pages, 0);
|
||||
@ -179,6 +181,7 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev)
|
||||
atomic64_set(&ctrs->sync_map_rpcits, 0);
|
||||
atomic64_set(&ctrs->sync_rpcits, 0);
|
||||
}
|
||||
spin_unlock_irqrestore(&zdev->dom_lock, flags);
|
||||
|
||||
|
||||
fib.fmb_addr = virt_to_phys(zdev->fmb);
|
||||
|
@ -71,17 +71,23 @@ static void pci_fmb_show(struct seq_file *m, char *name[], int length,
|
||||
|
||||
static void pci_sw_counter_show(struct seq_file *m)
|
||||
{
|
||||
struct zpci_iommu_ctrs *ctrs = zpci_get_iommu_ctrs(m->private);
|
||||
struct zpci_dev *zdev = m->private;
|
||||
struct zpci_iommu_ctrs *ctrs;
|
||||
atomic64_t *counter;
|
||||
unsigned long flags;
|
||||
int i;
|
||||
|
||||
spin_lock_irqsave(&zdev->dom_lock, flags);
|
||||
ctrs = zpci_get_iommu_ctrs(m->private);
|
||||
if (!ctrs)
|
||||
return;
|
||||
goto unlock;
|
||||
|
||||
counter = &ctrs->mapped_pages;
|
||||
for (i = 0; i < ARRAY_SIZE(pci_sw_names); i++, counter++)
|
||||
seq_printf(m, "%26s:\t%llu\n", pci_sw_names[i],
|
||||
atomic64_read(counter));
|
||||
unlock:
|
||||
spin_unlock_irqrestore(&zdev->dom_lock, flags);
|
||||
}
|
||||
|
||||
static int pci_perf_show(struct seq_file *m, void *v)
|
||||
|
@ -120,8 +120,8 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev)
|
||||
mutex_init(&tdev->iommu.mutex);
|
||||
|
||||
if (device_iommu_mapped(dev)) {
|
||||
tdev->iommu.domain = iommu_domain_alloc(&platform_bus_type);
|
||||
if (!tdev->iommu.domain)
|
||||
tdev->iommu.domain = iommu_paging_domain_alloc(dev);
|
||||
if (IS_ERR(tdev->iommu.domain))
|
||||
goto error;
|
||||
|
||||
/*
|
||||
|
@ -195,6 +195,7 @@ config MSM_IOMMU
|
||||
source "drivers/iommu/amd/Kconfig"
|
||||
source "drivers/iommu/intel/Kconfig"
|
||||
source "drivers/iommu/iommufd/Kconfig"
|
||||
source "drivers/iommu/riscv/Kconfig"
|
||||
|
||||
config IRQ_REMAP
|
||||
bool "Support for Interrupt Remapping"
|
||||
|
@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
obj-y += amd/ intel/ arm/ iommufd/
|
||||
obj-y += amd/ intel/ arm/ iommufd/ riscv/
|
||||
obj-$(CONFIG_IOMMU_API) += iommu.o
|
||||
obj-$(CONFIG_IOMMU_API) += iommu-traces.o
|
||||
obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
|
||||
|
@ -46,6 +46,7 @@ extern int amd_iommu_gpt_level;
|
||||
extern unsigned long amd_iommu_pgsize_bitmap;
|
||||
|
||||
/* Protection domain ops */
|
||||
void amd_iommu_init_identity_domain(void);
|
||||
struct protection_domain *protection_domain_alloc(unsigned int type, int nid);
|
||||
void protection_domain_free(struct protection_domain *domain);
|
||||
struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev,
|
||||
@ -118,9 +119,14 @@ static inline bool check_feature2(u64 mask)
|
||||
return (amd_iommu_efr2 & mask);
|
||||
}
|
||||
|
||||
static inline bool amd_iommu_v2_pgtbl_supported(void)
|
||||
{
|
||||
return (check_feature(FEATURE_GIOSUP) && check_feature(FEATURE_GT));
|
||||
}
|
||||
|
||||
static inline bool amd_iommu_gt_ppr_supported(void)
|
||||
{
|
||||
return (check_feature(FEATURE_GT) &&
|
||||
return (amd_iommu_v2_pgtbl_supported() &&
|
||||
check_feature(FEATURE_PPR) &&
|
||||
check_feature(FEATURE_EPHSUP));
|
||||
}
|
||||
|
@ -2070,14 +2070,6 @@ static int __init iommu_init_pci(struct amd_iommu *iommu)
|
||||
|
||||
init_iommu_perf_ctr(iommu);
|
||||
|
||||
if (amd_iommu_pgtable == AMD_IOMMU_V2) {
|
||||
if (!check_feature(FEATURE_GIOSUP) ||
|
||||
!check_feature(FEATURE_GT)) {
|
||||
pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
|
||||
amd_iommu_pgtable = AMD_IOMMU_V1;
|
||||
}
|
||||
}
|
||||
|
||||
if (is_rd890_iommu(iommu->dev)) {
|
||||
int i, j;
|
||||
|
||||
@ -2172,6 +2164,9 @@ static int __init amd_iommu_init_pci(void)
|
||||
struct amd_iommu_pci_seg *pci_seg;
|
||||
int ret;
|
||||
|
||||
/* Init global identity domain before registering IOMMU */
|
||||
amd_iommu_init_identity_domain();
|
||||
|
||||
for_each_iommu(iommu) {
|
||||
ret = iommu_init_pci(iommu);
|
||||
if (ret) {
|
||||
@ -3091,6 +3086,13 @@ static int __init early_amd_iommu_init(void)
|
||||
FIELD_GET(FEATURE_GATS, amd_iommu_efr) == GUEST_PGTABLE_5_LEVEL)
|
||||
amd_iommu_gpt_level = PAGE_MODE_5_LEVEL;
|
||||
|
||||
if (amd_iommu_pgtable == AMD_IOMMU_V2) {
|
||||
if (!amd_iommu_v2_pgtbl_supported()) {
|
||||
pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
|
||||
amd_iommu_pgtable = AMD_IOMMU_V1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Disable any previously enabled IOMMUs */
|
||||
if (!is_kdump_kernel() || amd_iommu_disabled)
|
||||
disable_iommus();
|
||||
|
@ -74,6 +74,9 @@ struct kmem_cache *amd_iommu_irq_cache;
|
||||
|
||||
static void detach_device(struct device *dev);
|
||||
|
||||
static int amd_iommu_attach_device(struct iommu_domain *dom,
|
||||
struct device *dev);
|
||||
|
||||
static void set_dte_entry(struct amd_iommu *iommu,
|
||||
struct iommu_dev_data *dev_data);
|
||||
|
||||
@ -2263,44 +2266,42 @@ void protection_domain_free(struct protection_domain *domain)
|
||||
kfree(domain);
|
||||
}
|
||||
|
||||
static void protection_domain_init(struct protection_domain *domain, int nid)
|
||||
{
|
||||
spin_lock_init(&domain->lock);
|
||||
INIT_LIST_HEAD(&domain->dev_list);
|
||||
INIT_LIST_HEAD(&domain->dev_data_list);
|
||||
domain->iop.pgtbl.cfg.amd.nid = nid;
|
||||
}
|
||||
|
||||
struct protection_domain *protection_domain_alloc(unsigned int type, int nid)
|
||||
{
|
||||
struct io_pgtable_ops *pgtbl_ops;
|
||||
struct protection_domain *domain;
|
||||
int pgtable;
|
||||
|
||||
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
|
||||
if (!domain)
|
||||
return NULL;
|
||||
|
||||
domain->id = domain_id_alloc();
|
||||
if (!domain->id)
|
||||
goto err_free;
|
||||
|
||||
spin_lock_init(&domain->lock);
|
||||
INIT_LIST_HEAD(&domain->dev_list);
|
||||
INIT_LIST_HEAD(&domain->dev_data_list);
|
||||
domain->iop.pgtbl.cfg.amd.nid = nid;
|
||||
|
||||
switch (type) {
|
||||
/* No need to allocate io pgtable ops in passthrough mode */
|
||||
case IOMMU_DOMAIN_IDENTITY:
|
||||
case IOMMU_DOMAIN_SVA:
|
||||
return domain;
|
||||
case IOMMU_DOMAIN_DMA:
|
||||
pgtable = amd_iommu_pgtable;
|
||||
break;
|
||||
/*
|
||||
* Force IOMMU v1 page table when allocating
|
||||
* domain for pass-through devices.
|
||||
*/
|
||||
case IOMMU_DOMAIN_UNMANAGED:
|
||||
pgtable = AMD_IOMMU_V1;
|
||||
break;
|
||||
default:
|
||||
goto err_id;
|
||||
if (!domain->id) {
|
||||
kfree(domain);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
protection_domain_init(domain, nid);
|
||||
|
||||
return domain;
|
||||
}
|
||||
|
||||
static int pdom_setup_pgtable(struct protection_domain *domain,
|
||||
unsigned int type, int pgtable)
|
||||
{
|
||||
struct io_pgtable_ops *pgtbl_ops;
|
||||
|
||||
/* No need to allocate io pgtable ops in passthrough mode */
|
||||
if (!(type & __IOMMU_DOMAIN_PAGING))
|
||||
return 0;
|
||||
|
||||
switch (pgtable) {
|
||||
case AMD_IOMMU_V1:
|
||||
domain->pd_mode = PD_MODE_V1;
|
||||
@ -2309,25 +2310,20 @@ struct protection_domain *protection_domain_alloc(unsigned int type, int nid)
|
||||
domain->pd_mode = PD_MODE_V2;
|
||||
break;
|
||||
default:
|
||||
goto err_id;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
pgtbl_ops =
|
||||
alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl.cfg, domain);
|
||||
if (!pgtbl_ops)
|
||||
goto err_id;
|
||||
return -ENOMEM;
|
||||
|
||||
return domain;
|
||||
err_id:
|
||||
domain_id_free(domain->id);
|
||||
err_free:
|
||||
kfree(domain);
|
||||
return NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline u64 dma_max_address(void)
|
||||
static inline u64 dma_max_address(int pgtable)
|
||||
{
|
||||
if (amd_iommu_pgtable == AMD_IOMMU_V1)
|
||||
if (pgtable == AMD_IOMMU_V1)
|
||||
return ~0ULL;
|
||||
|
||||
/* V2 with 4/5 level page table */
|
||||
@ -2340,11 +2336,13 @@ static bool amd_iommu_hd_support(struct amd_iommu *iommu)
|
||||
}
|
||||
|
||||
static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
|
||||
struct device *dev, u32 flags)
|
||||
struct device *dev,
|
||||
u32 flags, int pgtable)
|
||||
{
|
||||
bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
|
||||
struct protection_domain *domain;
|
||||
struct amd_iommu *iommu = NULL;
|
||||
int ret;
|
||||
|
||||
if (dev)
|
||||
iommu = get_amd_iommu_from_dev(dev);
|
||||
@ -2356,16 +2354,20 @@ static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
|
||||
if (amd_iommu_snp_en && (type == IOMMU_DOMAIN_IDENTITY))
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
if (dirty_tracking && !amd_iommu_hd_support(iommu))
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
domain = protection_domain_alloc(type,
|
||||
dev ? dev_to_node(dev) : NUMA_NO_NODE);
|
||||
if (!domain)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
ret = pdom_setup_pgtable(domain, type, pgtable);
|
||||
if (ret) {
|
||||
domain_id_free(domain->id);
|
||||
kfree(domain);
|
||||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
domain->domain.geometry.aperture_start = 0;
|
||||
domain->domain.geometry.aperture_end = dma_max_address();
|
||||
domain->domain.geometry.aperture_end = dma_max_address(pgtable);
|
||||
domain->domain.geometry.force_aperture = true;
|
||||
domain->domain.pgsize_bitmap = domain->iop.pgtbl.cfg.pgsize_bitmap;
|
||||
|
||||
@ -2383,8 +2385,16 @@ static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
|
||||
static struct iommu_domain *amd_iommu_domain_alloc(unsigned int type)
|
||||
{
|
||||
struct iommu_domain *domain;
|
||||
int pgtable = amd_iommu_pgtable;
|
||||
|
||||
domain = do_iommu_domain_alloc(type, NULL, 0);
|
||||
/*
|
||||
* Force IOMMU v1 page table when allocating
|
||||
* domain for pass-through devices.
|
||||
*/
|
||||
if (type == IOMMU_DOMAIN_UNMANAGED)
|
||||
pgtable = AMD_IOMMU_V1;
|
||||
|
||||
domain = do_iommu_domain_alloc(type, NULL, 0, pgtable);
|
||||
if (IS_ERR(domain))
|
||||
return NULL;
|
||||
|
||||
@ -2398,11 +2408,36 @@ amd_iommu_domain_alloc_user(struct device *dev, u32 flags,
|
||||
|
||||
{
|
||||
unsigned int type = IOMMU_DOMAIN_UNMANAGED;
|
||||
struct amd_iommu *iommu = NULL;
|
||||
const u32 supported_flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
|
||||
IOMMU_HWPT_ALLOC_PASID;
|
||||
|
||||
if ((flags & ~IOMMU_HWPT_ALLOC_DIRTY_TRACKING) || parent || user_data)
|
||||
if (dev)
|
||||
iommu = get_amd_iommu_from_dev(dev);
|
||||
|
||||
if ((flags & ~supported_flags) || parent || user_data)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
return do_iommu_domain_alloc(type, dev, flags);
|
||||
/* Allocate domain with v2 page table if IOMMU supports PASID. */
|
||||
if (flags & IOMMU_HWPT_ALLOC_PASID) {
|
||||
if (!amd_iommu_pasid_supported())
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
return do_iommu_domain_alloc(type, dev, flags, AMD_IOMMU_V2);
|
||||
}
|
||||
|
||||
/* Allocate domain with v1 page table for dirty tracking */
|
||||
if (flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) {
|
||||
if (iommu && amd_iommu_hd_support(iommu)) {
|
||||
return do_iommu_domain_alloc(type, dev,
|
||||
flags, AMD_IOMMU_V1);
|
||||
}
|
||||
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
}
|
||||
|
||||
/* If nothing specific is required use the kernel commandline default */
|
||||
return do_iommu_domain_alloc(type, dev, 0, amd_iommu_pgtable);
|
||||
}
|
||||
|
||||
void amd_iommu_domain_free(struct iommu_domain *dom)
|
||||
@ -2444,6 +2479,25 @@ static struct iommu_domain blocked_domain = {
|
||||
}
|
||||
};
|
||||
|
||||
static struct protection_domain identity_domain;
|
||||
|
||||
static const struct iommu_domain_ops identity_domain_ops = {
|
||||
.attach_dev = amd_iommu_attach_device,
|
||||
};
|
||||
|
||||
void amd_iommu_init_identity_domain(void)
|
||||
{
|
||||
struct iommu_domain *domain = &identity_domain.domain;
|
||||
|
||||
domain->type = IOMMU_DOMAIN_IDENTITY;
|
||||
domain->ops = &identity_domain_ops;
|
||||
domain->owner = &amd_iommu_ops;
|
||||
|
||||
identity_domain.id = domain_id_alloc();
|
||||
|
||||
protection_domain_init(&identity_domain, NUMA_NO_NODE);
|
||||
}
|
||||
|
||||
static int amd_iommu_attach_device(struct iommu_domain *dom,
|
||||
struct device *dev)
|
||||
{
|
||||
@ -2842,6 +2896,7 @@ static int amd_iommu_dev_disable_feature(struct device *dev,
|
||||
const struct iommu_ops amd_iommu_ops = {
|
||||
.capable = amd_iommu_capable,
|
||||
.blocked_domain = &blocked_domain,
|
||||
.identity_domain = &identity_domain.domain,
|
||||
.domain_alloc = amd_iommu_domain_alloc,
|
||||
.domain_alloc_user = amd_iommu_domain_alloc_user,
|
||||
.domain_alloc_sva = amd_iommu_domain_alloc_sva,
|
||||
|
@ -3084,7 +3084,8 @@ arm_smmu_domain_alloc_user(struct device *dev, u32 flags,
|
||||
const struct iommu_user_data *user_data)
|
||||
{
|
||||
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
|
||||
const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
|
||||
const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
|
||||
IOMMU_HWPT_ALLOC_PASID;
|
||||
struct arm_smmu_domain *smmu_domain;
|
||||
int ret;
|
||||
|
||||
@ -3093,6 +3094,9 @@ arm_smmu_domain_alloc_user(struct device *dev, u32 flags,
|
||||
if (parent || user_data)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
if (flags & IOMMU_HWPT_ALLOC_PASID)
|
||||
return arm_smmu_domain_alloc_paging(dev);
|
||||
|
||||
smmu_domain = arm_smmu_domain_alloc();
|
||||
if (IS_ERR(smmu_domain))
|
||||
return ERR_CAST(smmu_domain);
|
||||
|
@ -509,7 +509,8 @@ static int tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq *vcmdq)
|
||||
|
||||
snprintf(name, 16, "vcmdq%u", vcmdq->idx);
|
||||
|
||||
q->llq.max_n_shift = VCMDQ_LOG2SIZE_MAX;
|
||||
/* Queue size, capped to ensure natural alignment */
|
||||
q->llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, VCMDQ_LOG2SIZE_MAX);
|
||||
|
||||
/* Use the common helper to init the VCMDQ, and then... */
|
||||
ret = arm_smmu_init_one_queue(smmu, q, vcmdq->page0,
|
||||
@ -800,7 +801,9 @@ out_fallback:
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct dentry *cmdqv_debugfs_dir;
|
||||
#ifdef CONFIG_IOMMU_DEBUGFS
|
||||
static struct dentry *cmdqv_debugfs_dir;
|
||||
#endif
|
||||
|
||||
static struct arm_smmu_device *
|
||||
__tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
|
||||
|
@ -1437,6 +1437,17 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev)
|
||||
goto out_free;
|
||||
} else {
|
||||
smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
|
||||
|
||||
/*
|
||||
* Defer probe if the relevant SMMU instance hasn't finished
|
||||
* probing yet. This is a fragile hack and we'd ideally
|
||||
* avoid this race in the core code. Until that's ironed
|
||||
* out, however, this is the most pragmatic option on the
|
||||
* table.
|
||||
*/
|
||||
if (!smmu)
|
||||
return ERR_PTR(dev_err_probe(dev, -EPROBE_DEFER,
|
||||
"smmu dev has not bound yet\n"));
|
||||
}
|
||||
|
||||
ret = -EINVAL;
|
||||
|
@ -166,7 +166,6 @@ struct arm_v7s_io_pgtable {
|
||||
|
||||
arm_v7s_iopte *pgd;
|
||||
struct kmem_cache *l2_tables;
|
||||
spinlock_t split_lock;
|
||||
};
|
||||
|
||||
static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl);
|
||||
@ -363,25 +362,6 @@ static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl,
|
||||
return pte;
|
||||
}
|
||||
|
||||
static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl)
|
||||
{
|
||||
int prot = IOMMU_READ;
|
||||
arm_v7s_iopte attr = pte >> ARM_V7S_ATTR_SHIFT(lvl);
|
||||
|
||||
if (!(attr & ARM_V7S_PTE_AP_RDONLY))
|
||||
prot |= IOMMU_WRITE;
|
||||
if (!(attr & ARM_V7S_PTE_AP_UNPRIV))
|
||||
prot |= IOMMU_PRIV;
|
||||
if ((attr & (ARM_V7S_TEX_MASK << ARM_V7S_TEX_SHIFT)) == 0)
|
||||
prot |= IOMMU_MMIO;
|
||||
else if (pte & ARM_V7S_ATTR_C)
|
||||
prot |= IOMMU_CACHE;
|
||||
if (pte & ARM_V7S_ATTR_XN(lvl))
|
||||
prot |= IOMMU_NOEXEC;
|
||||
|
||||
return prot;
|
||||
}
|
||||
|
||||
static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl)
|
||||
{
|
||||
if (lvl == 1) {
|
||||
@ -398,23 +378,6 @@ static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl)
|
||||
return pte;
|
||||
}
|
||||
|
||||
static arm_v7s_iopte arm_v7s_cont_to_pte(arm_v7s_iopte pte, int lvl)
|
||||
{
|
||||
if (lvl == 1) {
|
||||
pte &= ~ARM_V7S_CONT_SECTION;
|
||||
} else if (lvl == 2) {
|
||||
arm_v7s_iopte xn = pte & BIT(ARM_V7S_CONT_PAGE_XN_SHIFT);
|
||||
arm_v7s_iopte tex = pte & (ARM_V7S_CONT_PAGE_TEX_MASK <<
|
||||
ARM_V7S_CONT_PAGE_TEX_SHIFT);
|
||||
|
||||
pte ^= xn | tex | ARM_V7S_PTE_TYPE_CONT_PAGE;
|
||||
pte |= (xn >> ARM_V7S_CONT_PAGE_XN_SHIFT) |
|
||||
(tex >> ARM_V7S_CONT_PAGE_TEX_SHIFT) |
|
||||
ARM_V7S_PTE_TYPE_PAGE;
|
||||
}
|
||||
return pte;
|
||||
}
|
||||
|
||||
static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl)
|
||||
{
|
||||
if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl))
|
||||
@ -591,77 +554,6 @@ static void arm_v7s_free_pgtable(struct io_pgtable *iop)
|
||||
kfree(data);
|
||||
}
|
||||
|
||||
static arm_v7s_iopte arm_v7s_split_cont(struct arm_v7s_io_pgtable *data,
|
||||
unsigned long iova, int idx, int lvl,
|
||||
arm_v7s_iopte *ptep)
|
||||
{
|
||||
struct io_pgtable *iop = &data->iop;
|
||||
arm_v7s_iopte pte;
|
||||
size_t size = ARM_V7S_BLOCK_SIZE(lvl);
|
||||
int i;
|
||||
|
||||
/* Check that we didn't lose a race to get the lock */
|
||||
pte = *ptep;
|
||||
if (!arm_v7s_pte_is_cont(pte, lvl))
|
||||
return pte;
|
||||
|
||||
ptep -= idx & (ARM_V7S_CONT_PAGES - 1);
|
||||
pte = arm_v7s_cont_to_pte(pte, lvl);
|
||||
for (i = 0; i < ARM_V7S_CONT_PAGES; i++)
|
||||
ptep[i] = pte + i * size;
|
||||
|
||||
__arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, &iop->cfg);
|
||||
|
||||
size *= ARM_V7S_CONT_PAGES;
|
||||
io_pgtable_tlb_flush_walk(iop, iova, size, size);
|
||||
return pte;
|
||||
}
|
||||
|
||||
static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data,
|
||||
struct iommu_iotlb_gather *gather,
|
||||
unsigned long iova, size_t size,
|
||||
arm_v7s_iopte blk_pte,
|
||||
arm_v7s_iopte *ptep)
|
||||
{
|
||||
struct io_pgtable_cfg *cfg = &data->iop.cfg;
|
||||
arm_v7s_iopte pte, *tablep;
|
||||
int i, unmap_idx, num_entries, num_ptes;
|
||||
|
||||
tablep = __arm_v7s_alloc_table(2, GFP_ATOMIC, data);
|
||||
if (!tablep)
|
||||
return 0; /* Bytes unmapped */
|
||||
|
||||
num_ptes = ARM_V7S_PTES_PER_LVL(2, cfg);
|
||||
num_entries = size >> ARM_V7S_LVL_SHIFT(2);
|
||||
unmap_idx = ARM_V7S_LVL_IDX(iova, 2, cfg);
|
||||
|
||||
pte = arm_v7s_prot_to_pte(arm_v7s_pte_to_prot(blk_pte, 1), 2, cfg);
|
||||
if (num_entries > 1)
|
||||
pte = arm_v7s_pte_to_cont(pte, 2);
|
||||
|
||||
for (i = 0; i < num_ptes; i += num_entries, pte += size) {
|
||||
/* Unmap! */
|
||||
if (i == unmap_idx)
|
||||
continue;
|
||||
|
||||
__arm_v7s_set_pte(&tablep[i], pte, num_entries, cfg);
|
||||
}
|
||||
|
||||
pte = arm_v7s_install_table(tablep, ptep, blk_pte, cfg);
|
||||
if (pte != blk_pte) {
|
||||
__arm_v7s_free_table(tablep, 2, data);
|
||||
|
||||
if (!ARM_V7S_PTE_IS_TABLE(pte, 1))
|
||||
return 0;
|
||||
|
||||
tablep = iopte_deref(pte, 1, data);
|
||||
return __arm_v7s_unmap(data, gather, iova, size, 2, tablep);
|
||||
}
|
||||
|
||||
io_pgtable_tlb_add_page(&data->iop, gather, iova, size);
|
||||
return size;
|
||||
}
|
||||
|
||||
static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
|
||||
struct iommu_iotlb_gather *gather,
|
||||
unsigned long iova, size_t size, int lvl,
|
||||
@ -694,11 +586,8 @@ static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
|
||||
* case in a lock for the sake of correctness and be done with it.
|
||||
*/
|
||||
if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) {
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&data->split_lock, flags);
|
||||
pte[0] = arm_v7s_split_cont(data, iova, idx, lvl, ptep);
|
||||
spin_unlock_irqrestore(&data->split_lock, flags);
|
||||
WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* If the size matches this level, we're in the right place */
|
||||
@ -721,12 +610,8 @@ static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
|
||||
}
|
||||
return size;
|
||||
} else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) {
|
||||
/*
|
||||
* Insert a table at the next level to map the old region,
|
||||
* minus the part we want to unmap
|
||||
*/
|
||||
return arm_v7s_split_blk_unmap(data, gather, iova, size, pte[0],
|
||||
ptep);
|
||||
WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Keep on walkin' */
|
||||
@ -811,8 +696,6 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,
|
||||
if (!data)
|
||||
return NULL;
|
||||
|
||||
spin_lock_init(&data->split_lock);
|
||||
|
||||
/*
|
||||
* ARM_MTK_TTBR_EXT extend the translation table base support larger
|
||||
* memory address.
|
||||
@ -936,8 +819,8 @@ static int __init arm_v7s_do_selftests(void)
|
||||
.quirks = IO_PGTABLE_QUIRK_ARM_NS,
|
||||
.pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
|
||||
};
|
||||
unsigned int iova, size, iova_start;
|
||||
unsigned int i, loopnr = 0;
|
||||
unsigned int iova, size;
|
||||
unsigned int i;
|
||||
size_t mapped;
|
||||
|
||||
selftest_running = true;
|
||||
@ -985,26 +868,6 @@ static int __init arm_v7s_do_selftests(void)
|
||||
return __FAIL(ops);
|
||||
|
||||
iova += SZ_16M;
|
||||
loopnr++;
|
||||
}
|
||||
|
||||
/* Partial unmap */
|
||||
i = 1;
|
||||
size = 1UL << __ffs(cfg.pgsize_bitmap);
|
||||
while (i < loopnr) {
|
||||
iova_start = i * SZ_16M;
|
||||
if (ops->unmap_pages(ops, iova_start + size, size, 1, NULL) != size)
|
||||
return __FAIL(ops);
|
||||
|
||||
/* Remap of partial unmap */
|
||||
if (ops->map_pages(ops, iova_start + size, size, size, 1,
|
||||
IOMMU_READ, GFP_KERNEL, &mapped))
|
||||
return __FAIL(ops);
|
||||
|
||||
if (ops->iova_to_phys(ops, iova_start + size + 42)
|
||||
!= (size + 42))
|
||||
return __FAIL(ops);
|
||||
i++;
|
||||
}
|
||||
|
||||
/* Full unmap */
|
||||
|
@ -199,6 +199,18 @@ static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte,
|
||||
return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert an index returned by ARM_LPAE_PGD_IDX(), which can point into
|
||||
* a concatenated PGD, into the maximum number of entries that can be
|
||||
* mapped in the same table page.
|
||||
*/
|
||||
static inline int arm_lpae_max_entries(int i, struct arm_lpae_io_pgtable *data)
|
||||
{
|
||||
int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data);
|
||||
|
||||
return ptes_per_table - (i & (ptes_per_table - 1));
|
||||
}
|
||||
|
||||
static bool selftest_running = false;
|
||||
|
||||
static dma_addr_t __arm_lpae_dma_addr(void *pages)
|
||||
@ -390,7 +402,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
|
||||
|
||||
/* If we can install a leaf entry at this level, then do so */
|
||||
if (size == block_size) {
|
||||
max_entries = ARM_LPAE_PTES_PER_TABLE(data) - map_idx_start;
|
||||
max_entries = arm_lpae_max_entries(map_idx_start, data);
|
||||
num_entries = min_t(int, pgcount, max_entries);
|
||||
ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
|
||||
if (!ret)
|
||||
@ -569,66 +581,6 @@ static void arm_lpae_free_pgtable(struct io_pgtable *iop)
|
||||
kfree(data);
|
||||
}
|
||||
|
||||
static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
|
||||
struct iommu_iotlb_gather *gather,
|
||||
unsigned long iova, size_t size,
|
||||
arm_lpae_iopte blk_pte, int lvl,
|
||||
arm_lpae_iopte *ptep, size_t pgcount)
|
||||
{
|
||||
struct io_pgtable_cfg *cfg = &data->iop.cfg;
|
||||
arm_lpae_iopte pte, *tablep;
|
||||
phys_addr_t blk_paddr;
|
||||
size_t tablesz = ARM_LPAE_GRANULE(data);
|
||||
size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
|
||||
int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data);
|
||||
int i, unmap_idx_start = -1, num_entries = 0, max_entries;
|
||||
|
||||
if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
|
||||
return 0;
|
||||
|
||||
tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg, data->iop.cookie);
|
||||
if (!tablep)
|
||||
return 0; /* Bytes unmapped */
|
||||
|
||||
if (size == split_sz) {
|
||||
unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
|
||||
max_entries = ptes_per_table - unmap_idx_start;
|
||||
num_entries = min_t(int, pgcount, max_entries);
|
||||
}
|
||||
|
||||
blk_paddr = iopte_to_paddr(blk_pte, data);
|
||||
pte = iopte_prot(blk_pte);
|
||||
|
||||
for (i = 0; i < ptes_per_table; i++, blk_paddr += split_sz) {
|
||||
/* Unmap! */
|
||||
if (i >= unmap_idx_start && i < (unmap_idx_start + num_entries))
|
||||
continue;
|
||||
|
||||
__arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i]);
|
||||
}
|
||||
|
||||
pte = arm_lpae_install_table(tablep, ptep, blk_pte, data);
|
||||
if (pte != blk_pte) {
|
||||
__arm_lpae_free_pages(tablep, tablesz, cfg, data->iop.cookie);
|
||||
/*
|
||||
* We may race against someone unmapping another part of this
|
||||
* block, but anything else is invalid. We can't misinterpret
|
||||
* a page entry here since we're never at the last level.
|
||||
*/
|
||||
if (iopte_type(pte) != ARM_LPAE_PTE_TYPE_TABLE)
|
||||
return 0;
|
||||
|
||||
tablep = iopte_deref(pte, data);
|
||||
} else if (unmap_idx_start >= 0) {
|
||||
for (i = 0; i < num_entries; i++)
|
||||
io_pgtable_tlb_add_page(&data->iop, gather, iova + i * size, size);
|
||||
|
||||
return num_entries * size;
|
||||
}
|
||||
|
||||
return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl, tablep);
|
||||
}
|
||||
|
||||
static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
|
||||
struct iommu_iotlb_gather *gather,
|
||||
unsigned long iova, size_t size, size_t pgcount,
|
||||
@ -650,7 +602,7 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
|
||||
|
||||
/* If the size matches this level, we're in the right place */
|
||||
if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
|
||||
max_entries = ARM_LPAE_PTES_PER_TABLE(data) - unmap_idx_start;
|
||||
max_entries = arm_lpae_max_entries(unmap_idx_start, data);
|
||||
num_entries = min_t(int, pgcount, max_entries);
|
||||
|
||||
/* Find and handle non-leaf entries */
|
||||
@ -678,12 +630,8 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
|
||||
|
||||
return i * size;
|
||||
} else if (iopte_leaf(pte, lvl, iop->fmt)) {
|
||||
/*
|
||||
* Insert a table at the next level to map the old region,
|
||||
* minus the part we want to unmap
|
||||
*/
|
||||
return arm_lpae_split_blk_unmap(data, gather, iova, size, pte,
|
||||
lvl + 1, ptep, pgcount);
|
||||
WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Keep on walkin' */
|
||||
@ -1347,19 +1295,6 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
|
||||
iova += SZ_1G;
|
||||
}
|
||||
|
||||
/* Partial unmap */
|
||||
size = 1UL << __ffs(cfg->pgsize_bitmap);
|
||||
if (ops->unmap_pages(ops, SZ_1G + size, size, 1, NULL) != size)
|
||||
return __FAIL(ops, i);
|
||||
|
||||
/* Remap of partial unmap */
|
||||
if (ops->map_pages(ops, SZ_1G + size, size, size, 1,
|
||||
IOMMU_READ, GFP_KERNEL, &mapped))
|
||||
return __FAIL(ops, i);
|
||||
|
||||
if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42))
|
||||
return __FAIL(ops, i);
|
||||
|
||||
/* Full unmap */
|
||||
iova = 0;
|
||||
for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
|
||||
@ -1382,6 +1317,23 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
|
||||
iova += SZ_1G;
|
||||
}
|
||||
|
||||
/*
|
||||
* Map/unmap the last largest supported page of the IAS, this can
|
||||
* trigger corner cases in the concatednated page tables.
|
||||
*/
|
||||
mapped = 0;
|
||||
size = 1UL << __fls(cfg->pgsize_bitmap);
|
||||
iova = (1UL << cfg->ias) - size;
|
||||
if (ops->map_pages(ops, iova, iova, size, 1,
|
||||
IOMMU_READ | IOMMU_WRITE |
|
||||
IOMMU_NOEXEC | IOMMU_CACHE,
|
||||
GFP_KERNEL, &mapped))
|
||||
return __FAIL(ops, i);
|
||||
if (mapped != size)
|
||||
return __FAIL(ops, i);
|
||||
if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
|
||||
return __FAIL(ops, i);
|
||||
|
||||
free_io_pgtable_ops(ops);
|
||||
}
|
||||
|
||||
|
@ -34,7 +34,7 @@ static void release_device(struct device *dev)
|
||||
kfree(dev);
|
||||
}
|
||||
|
||||
static struct class iommu_class = {
|
||||
static const struct class iommu_class = {
|
||||
.name = "iommu",
|
||||
.dev_release = release_device,
|
||||
.dev_groups = dev_groups,
|
||||
|
@ -32,6 +32,7 @@
|
||||
#include <trace/events/iommu.h>
|
||||
#include <linux/sched/mm.h>
|
||||
#include <linux/msi.h>
|
||||
#include <uapi/linux/iommufd.h>
|
||||
|
||||
#include "dma-iommu.h"
|
||||
#include "iommu-priv.h"
|
||||
@ -90,15 +91,17 @@ static const char * const iommu_group_resv_type_string[] = {
|
||||
#define IOMMU_CMD_LINE_DMA_API BIT(0)
|
||||
#define IOMMU_CMD_LINE_STRICT BIT(1)
|
||||
|
||||
static int bus_iommu_probe(const struct bus_type *bus);
|
||||
static int iommu_bus_notifier(struct notifier_block *nb,
|
||||
unsigned long action, void *data);
|
||||
static void iommu_release_device(struct device *dev);
|
||||
static struct iommu_domain *
|
||||
__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type);
|
||||
static int __iommu_attach_device(struct iommu_domain *domain,
|
||||
struct device *dev);
|
||||
static int __iommu_attach_group(struct iommu_domain *domain,
|
||||
struct iommu_group *group);
|
||||
static struct iommu_domain *__iommu_paging_domain_alloc_flags(struct device *dev,
|
||||
unsigned int type,
|
||||
unsigned int flags);
|
||||
|
||||
enum {
|
||||
IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0,
|
||||
@ -133,6 +136,8 @@ static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
|
||||
struct device *dev);
|
||||
static void __iommu_group_free_device(struct iommu_group *group,
|
||||
struct group_device *grp_dev);
|
||||
static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
|
||||
const struct iommu_ops *ops);
|
||||
|
||||
#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
|
||||
struct iommu_group_attribute iommu_group_attr_##_name = \
|
||||
@ -1141,10 +1146,6 @@ map_end:
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (!list_empty(&mappings) && iommu_is_dma_domain(domain))
|
||||
iommu_flush_iotlb_all(domain);
|
||||
|
||||
out:
|
||||
iommu_put_resv_regions(dev, &mappings);
|
||||
|
||||
@ -1586,12 +1587,59 @@ struct iommu_group *fsl_mc_device_group(struct device *dev)
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(fsl_mc_device_group);
|
||||
|
||||
static struct iommu_domain *__iommu_alloc_identity_domain(struct device *dev)
|
||||
{
|
||||
const struct iommu_ops *ops = dev_iommu_ops(dev);
|
||||
struct iommu_domain *domain;
|
||||
|
||||
if (ops->identity_domain)
|
||||
return ops->identity_domain;
|
||||
|
||||
/* Older drivers create the identity domain via ops->domain_alloc() */
|
||||
if (!ops->domain_alloc)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
domain = ops->domain_alloc(IOMMU_DOMAIN_IDENTITY);
|
||||
if (IS_ERR(domain))
|
||||
return domain;
|
||||
if (!domain)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
iommu_domain_init(domain, IOMMU_DOMAIN_IDENTITY, ops);
|
||||
return domain;
|
||||
}
|
||||
|
||||
static struct iommu_domain *
|
||||
__iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
|
||||
{
|
||||
struct device *dev = iommu_group_first_dev(group);
|
||||
struct iommu_domain *dom;
|
||||
|
||||
if (group->default_domain && group->default_domain->type == req_type)
|
||||
return group->default_domain;
|
||||
return __iommu_group_domain_alloc(group, req_type);
|
||||
|
||||
/*
|
||||
* When allocating the DMA API domain assume that the driver is going to
|
||||
* use PASID and make sure the RID's domain is PASID compatible.
|
||||
*/
|
||||
if (req_type & __IOMMU_DOMAIN_PAGING) {
|
||||
dom = __iommu_paging_domain_alloc_flags(dev, req_type,
|
||||
dev->iommu->max_pasids ? IOMMU_HWPT_ALLOC_PASID : 0);
|
||||
|
||||
/*
|
||||
* If driver does not support PASID feature then
|
||||
* try to allocate non-PASID domain
|
||||
*/
|
||||
if (PTR_ERR(dom) == -EOPNOTSUPP)
|
||||
dom = __iommu_paging_domain_alloc_flags(dev, req_type, 0);
|
||||
|
||||
return dom;
|
||||
}
|
||||
|
||||
if (req_type == IOMMU_DOMAIN_IDENTITY)
|
||||
return __iommu_alloc_identity_domain(dev);
|
||||
|
||||
return ERR_PTR(-EINVAL);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1795,7 +1843,7 @@ static void iommu_group_do_probe_finalize(struct device *dev)
|
||||
ops->probe_finalize(dev);
|
||||
}
|
||||
|
||||
int bus_iommu_probe(const struct bus_type *bus)
|
||||
static int bus_iommu_probe(const struct bus_type *bus)
|
||||
{
|
||||
struct iommu_group *group, *next;
|
||||
LIST_HEAD(group_list);
|
||||
@ -1840,31 +1888,6 @@ int bus_iommu_probe(const struct bus_type *bus)
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* iommu_present() - make platform-specific assumptions about an IOMMU
|
||||
* @bus: bus to check
|
||||
*
|
||||
* Do not use this function. You want device_iommu_mapped() instead.
|
||||
*
|
||||
* Return: true if some IOMMU is present and aware of devices on the given bus;
|
||||
* in general it may not be the only IOMMU, and it may not have anything to do
|
||||
* with whatever device you are ultimately interested in.
|
||||
*/
|
||||
bool iommu_present(const struct bus_type *bus)
|
||||
{
|
||||
bool ret = false;
|
||||
|
||||
for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
|
||||
if (iommu_buses[i] == bus) {
|
||||
spin_lock(&iommu_device_lock);
|
||||
ret = !list_empty(&iommu_device_list);
|
||||
spin_unlock(&iommu_device_lock);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(iommu_present);
|
||||
|
||||
/**
|
||||
* device_iommu_capable() - check for a general IOMMU capability
|
||||
* @dev: device to which the capability would be relevant, if available
|
||||
@ -1934,117 +1957,67 @@ void iommu_set_fault_handler(struct iommu_domain *domain,
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
|
||||
|
||||
static struct iommu_domain *__iommu_domain_alloc(const struct iommu_ops *ops,
|
||||
struct device *dev,
|
||||
unsigned int type)
|
||||
static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
|
||||
const struct iommu_ops *ops)
|
||||
{
|
||||
struct iommu_domain *domain;
|
||||
unsigned int alloc_type = type & IOMMU_DOMAIN_ALLOC_FLAGS;
|
||||
|
||||
if (alloc_type == IOMMU_DOMAIN_IDENTITY && ops->identity_domain)
|
||||
return ops->identity_domain;
|
||||
else if (alloc_type == IOMMU_DOMAIN_BLOCKED && ops->blocked_domain)
|
||||
return ops->blocked_domain;
|
||||
else if (type & __IOMMU_DOMAIN_PAGING && ops->domain_alloc_paging)
|
||||
domain = ops->domain_alloc_paging(dev);
|
||||
else if (ops->domain_alloc)
|
||||
domain = ops->domain_alloc(alloc_type);
|
||||
else
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
/*
|
||||
* Many domain_alloc ops now return ERR_PTR, make things easier for the
|
||||
* driver by accepting ERR_PTR from all domain_alloc ops instead of
|
||||
* having two rules.
|
||||
*/
|
||||
if (IS_ERR(domain))
|
||||
return domain;
|
||||
if (!domain)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
domain->type = type;
|
||||
domain->owner = ops;
|
||||
if (!domain->ops)
|
||||
domain->ops = ops->default_domain_ops;
|
||||
|
||||
/*
|
||||
* If not already set, assume all sizes by default; the driver
|
||||
* may override this later
|
||||
*/
|
||||
if (!domain->pgsize_bitmap)
|
||||
domain->pgsize_bitmap = ops->pgsize_bitmap;
|
||||
|
||||
if (!domain->ops)
|
||||
domain->ops = ops->default_domain_ops;
|
||||
|
||||
if (iommu_is_dma_domain(domain)) {
|
||||
int rc;
|
||||
|
||||
rc = iommu_get_dma_cookie(domain);
|
||||
if (rc) {
|
||||
iommu_domain_free(domain);
|
||||
return ERR_PTR(rc);
|
||||
}
|
||||
}
|
||||
return domain;
|
||||
}
|
||||
|
||||
static struct iommu_domain *
|
||||
__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type)
|
||||
__iommu_paging_domain_alloc_flags(struct device *dev, unsigned int type,
|
||||
unsigned int flags)
|
||||
{
|
||||
struct device *dev = iommu_group_first_dev(group);
|
||||
|
||||
return __iommu_domain_alloc(dev_iommu_ops(dev), dev, type);
|
||||
}
|
||||
|
||||
static int __iommu_domain_alloc_dev(struct device *dev, void *data)
|
||||
{
|
||||
const struct iommu_ops **ops = data;
|
||||
|
||||
if (!dev_has_iommu(dev))
|
||||
return 0;
|
||||
|
||||
if (WARN_ONCE(*ops && *ops != dev_iommu_ops(dev),
|
||||
"Multiple IOMMU drivers present for bus %s, which the public IOMMU API can't fully support yet. You will still need to disable one or more for this to work, sorry!\n",
|
||||
dev_bus_name(dev)))
|
||||
return -EBUSY;
|
||||
|
||||
*ops = dev_iommu_ops(dev);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* The iommu ops in bus has been retired. Do not use this interface in
|
||||
* new drivers.
|
||||
*/
|
||||
struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
|
||||
{
|
||||
const struct iommu_ops *ops = NULL;
|
||||
int err = bus_for_each_dev(bus, NULL, &ops, __iommu_domain_alloc_dev);
|
||||
const struct iommu_ops *ops;
|
||||
struct iommu_domain *domain;
|
||||
|
||||
if (err || !ops)
|
||||
return NULL;
|
||||
|
||||
domain = __iommu_domain_alloc(ops, NULL, IOMMU_DOMAIN_UNMANAGED);
|
||||
if (IS_ERR(domain))
|
||||
return NULL;
|
||||
return domain;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(iommu_domain_alloc);
|
||||
|
||||
/**
|
||||
* iommu_paging_domain_alloc() - Allocate a paging domain
|
||||
* @dev: device for which the domain is allocated
|
||||
*
|
||||
* Allocate a paging domain which will be managed by a kernel driver. Return
|
||||
* allocated domain if successful, or a ERR pointer for failure.
|
||||
*/
|
||||
struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
|
||||
{
|
||||
if (!dev_has_iommu(dev))
|
||||
return ERR_PTR(-ENODEV);
|
||||
|
||||
return __iommu_domain_alloc(dev_iommu_ops(dev), dev, IOMMU_DOMAIN_UNMANAGED);
|
||||
ops = dev_iommu_ops(dev);
|
||||
|
||||
if (ops->domain_alloc_paging && !flags)
|
||||
domain = ops->domain_alloc_paging(dev);
|
||||
else if (ops->domain_alloc_user)
|
||||
domain = ops->domain_alloc_user(dev, flags, NULL, NULL);
|
||||
else if (ops->domain_alloc && !flags)
|
||||
domain = ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED);
|
||||
else
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
if (IS_ERR(domain))
|
||||
return domain;
|
||||
if (!domain)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
iommu_domain_init(domain, type, ops);
|
||||
return domain;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc);
|
||||
|
||||
/**
|
||||
* iommu_paging_domain_alloc_flags() - Allocate a paging domain
|
||||
* @dev: device for which the domain is allocated
|
||||
* @flags: Bitmap of iommufd_hwpt_alloc_flags
|
||||
*
|
||||
* Allocate a paging domain which will be managed by a kernel driver. Return
|
||||
* allocated domain if successful, or an ERR pointer for failure.
|
||||
*/
|
||||
struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev,
|
||||
unsigned int flags)
|
||||
{
|
||||
return __iommu_paging_domain_alloc_flags(dev,
|
||||
IOMMU_DOMAIN_UNMANAGED, flags);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc_flags);
|
||||
|
||||
void iommu_domain_free(struct iommu_domain *domain)
|
||||
{
|
||||
@ -2216,8 +2189,8 @@ EXPORT_SYMBOL_GPL(iommu_attach_group);
|
||||
|
||||
/**
|
||||
* iommu_group_replace_domain - replace the domain that a group is attached to
|
||||
* @new_domain: new IOMMU domain to replace with
|
||||
* @group: IOMMU group that will be attached to the new domain
|
||||
* @new_domain: new IOMMU domain to replace with
|
||||
*
|
||||
* This API allows the group to switch domains without being forced to go to
|
||||
* the blocking domain in-between.
|
||||
@ -2586,6 +2559,20 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
|
||||
return unmapped;
|
||||
}
|
||||
|
||||
/**
|
||||
* iommu_unmap() - Remove mappings from a range of IOVA
|
||||
* @domain: Domain to manipulate
|
||||
* @iova: IO virtual address to start
|
||||
* @size: Length of the range starting from @iova
|
||||
*
|
||||
* iommu_unmap() will remove a translation created by iommu_map(). It cannot
|
||||
* subdivide a mapping created by iommu_map(), so it should be called with IOVA
|
||||
* ranges that match what was passed to iommu_map(). The range can aggregate
|
||||
* contiguous iommu_map() calls so long as no individual range is split.
|
||||
*
|
||||
* Returns: Number of bytes of IOVA unmapped. iova + res will be the point
|
||||
* unmapping stopped.
|
||||
*/
|
||||
size_t iommu_unmap(struct iommu_domain *domain,
|
||||
unsigned long iova, size_t size)
|
||||
{
|
||||
@ -2965,6 +2952,14 @@ static int iommu_setup_default_domain(struct iommu_group *group,
|
||||
if (group->default_domain == dom)
|
||||
return 0;
|
||||
|
||||
if (iommu_is_dma_domain(dom)) {
|
||||
ret = iommu_get_dma_cookie(dom);
|
||||
if (ret) {
|
||||
iommu_domain_free(dom);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* IOMMU_RESV_DIRECT and IOMMU_RESV_DIRECT_RELAXABLE regions must be
|
||||
* mapped before their device is attached, in order to guarantee
|
||||
@ -3152,22 +3147,25 @@ void iommu_device_unuse_default_domain(struct device *dev)
|
||||
|
||||
static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
|
||||
{
|
||||
struct device *dev = iommu_group_first_dev(group);
|
||||
const struct iommu_ops *ops = dev_iommu_ops(dev);
|
||||
struct iommu_domain *domain;
|
||||
|
||||
if (group->blocking_domain)
|
||||
return 0;
|
||||
|
||||
domain = __iommu_group_domain_alloc(group, IOMMU_DOMAIN_BLOCKED);
|
||||
if (IS_ERR(domain)) {
|
||||
/*
|
||||
* For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED
|
||||
* create an empty domain instead.
|
||||
*/
|
||||
domain = __iommu_group_domain_alloc(group,
|
||||
IOMMU_DOMAIN_UNMANAGED);
|
||||
if (IS_ERR(domain))
|
||||
return PTR_ERR(domain);
|
||||
if (ops->blocked_domain) {
|
||||
group->blocking_domain = ops->blocked_domain;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED create an
|
||||
* empty PAGING domain instead.
|
||||
*/
|
||||
domain = iommu_paging_domain_alloc(dev);
|
||||
if (IS_ERR(domain))
|
||||
return PTR_ERR(domain);
|
||||
group->blocking_domain = domain;
|
||||
return 0;
|
||||
}
|
||||
|
@ -506,7 +506,7 @@ __adjust_overlap_range(struct iova *iova,
|
||||
* reserve_iova - reserves an iova in the given range
|
||||
* @iovad: - iova domain pointer
|
||||
* @pfn_lo: - lower page frame address
|
||||
* @pfn_hi:- higher pfn adderss
|
||||
* @pfn_hi:- higher pfn address
|
||||
* This function allocates reserves the address range from pfn_lo to pfn_hi so
|
||||
* that this address is not dished out as part of alloc_iova.
|
||||
*/
|
||||
|
@ -1599,7 +1599,7 @@ static const unsigned int mt8186_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK
|
||||
static const struct mtk_iommu_plat_data mt8186_data_mm = {
|
||||
.m4u_plat = M4U_MT8186,
|
||||
.flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN |
|
||||
WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM,
|
||||
WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM | PGTABLE_PA_35_EN,
|
||||
.larbid_remap = {{0}, {1, MTK_INVALID_LARBID, 8}, {4}, {7}, {2}, {9, 11, 19, 20},
|
||||
{MTK_INVALID_LARBID, 14, 16},
|
||||
{MTK_INVALID_LARBID, 13, MTK_INVALID_LARBID, 17}},
|
||||
|
@ -1230,25 +1230,24 @@ static int omap_iommu_probe(struct platform_device *pdev)
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
err = iommu_device_register(&obj->iommu, &omap_iommu_ops, &pdev->dev);
|
||||
if (err)
|
||||
goto out_sysfs;
|
||||
obj->has_iommu_driver = true;
|
||||
}
|
||||
|
||||
err = iommu_device_register(&obj->iommu, &omap_iommu_ops, &pdev->dev);
|
||||
if (err)
|
||||
goto out_sysfs;
|
||||
|
||||
pm_runtime_enable(obj->dev);
|
||||
|
||||
omap_iommu_debugfs_add(obj);
|
||||
|
||||
dev_info(&pdev->dev, "%s registered\n", obj->name);
|
||||
|
||||
/* Re-probe bus to probe device attached to this IOMMU */
|
||||
bus_iommu_probe(&platform_bus_type);
|
||||
|
||||
return 0;
|
||||
|
||||
out_sysfs:
|
||||
iommu_device_sysfs_remove(&obj->iommu);
|
||||
if (obj->has_iommu_driver)
|
||||
iommu_device_sysfs_remove(&obj->iommu);
|
||||
return err;
|
||||
}
|
||||
|
||||
@ -1256,10 +1255,10 @@ static void omap_iommu_remove(struct platform_device *pdev)
|
||||
{
|
||||
struct omap_iommu *obj = platform_get_drvdata(pdev);
|
||||
|
||||
if (obj->has_iommu_driver) {
|
||||
if (obj->has_iommu_driver)
|
||||
iommu_device_sysfs_remove(&obj->iommu);
|
||||
iommu_device_unregister(&obj->iommu);
|
||||
}
|
||||
|
||||
iommu_device_unregister(&obj->iommu);
|
||||
|
||||
omap_iommu_debugfs_remove(obj);
|
||||
|
||||
@ -1723,12 +1722,19 @@ static void omap_iommu_release_device(struct device *dev)
|
||||
|
||||
}
|
||||
|
||||
static int omap_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
|
||||
{
|
||||
/* TODO: collect args->np to save re-parsing in probe above */
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct iommu_ops omap_iommu_ops = {
|
||||
.identity_domain = &omap_iommu_identity_domain,
|
||||
.domain_alloc_paging = omap_iommu_domain_alloc_paging,
|
||||
.probe_device = omap_iommu_probe_device,
|
||||
.release_device = omap_iommu_release_device,
|
||||
.device_group = generic_single_device_group,
|
||||
.of_xlate = omap_iommu_of_xlate,
|
||||
.pgsize_bitmap = OMAP_IOMMU_PGSIZES,
|
||||
.default_domain_ops = &(const struct iommu_domain_ops) {
|
||||
.attach_dev = omap_iommu_attach_dev,
|
||||
|
20
drivers/iommu/riscv/Kconfig
Normal file
20
drivers/iommu/riscv/Kconfig
Normal file
@ -0,0 +1,20 @@
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
# RISC-V IOMMU support
|
||||
|
||||
config RISCV_IOMMU
|
||||
bool "RISC-V IOMMU Support"
|
||||
depends on RISCV && 64BIT
|
||||
default y
|
||||
select IOMMU_API
|
||||
help
|
||||
Support for implementations of the RISC-V IOMMU architecture that
|
||||
complements the RISC-V MMU capabilities, providing similar address
|
||||
translation and protection functions for accesses from I/O devices.
|
||||
|
||||
Say Y here if your SoC includes an IOMMU device implementing
|
||||
the RISC-V IOMMU architecture.
|
||||
|
||||
config RISCV_IOMMU_PCI
|
||||
def_bool y if RISCV_IOMMU && PCI_MSI
|
||||
help
|
||||
Support for the PCIe implementation of RISC-V IOMMU architecture.
|
3
drivers/iommu/riscv/Makefile
Normal file
3
drivers/iommu/riscv/Makefile
Normal file
@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
obj-$(CONFIG_RISCV_IOMMU) += iommu.o iommu-platform.o
|
||||
obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o
|
784
drivers/iommu/riscv/iommu-bits.h
Normal file
784
drivers/iommu/riscv/iommu-bits.h
Normal file
@ -0,0 +1,784 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright © 2022-2024 Rivos Inc.
|
||||
* Copyright © 2023 FORTH-ICS/CARV
|
||||
* Copyright © 2023 RISC-V IOMMU Task Group
|
||||
*
|
||||
* RISC-V IOMMU - Register Layout and Data Structures.
|
||||
*
|
||||
* Based on the 'RISC-V IOMMU Architecture Specification', Version 1.0
|
||||
* Published at https://github.com/riscv-non-isa/riscv-iommu
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef _RISCV_IOMMU_BITS_H_
|
||||
#define _RISCV_IOMMU_BITS_H_
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/bits.h>
|
||||
|
||||
/*
|
||||
* Chapter 5: Memory Mapped register interface
|
||||
*/
|
||||
|
||||
/* Common field positions */
|
||||
#define RISCV_IOMMU_PPN_FIELD GENMASK_ULL(53, 10)
|
||||
#define RISCV_IOMMU_QUEUE_LOG2SZ_FIELD GENMASK_ULL(4, 0)
|
||||
#define RISCV_IOMMU_QUEUE_INDEX_FIELD GENMASK_ULL(31, 0)
|
||||
#define RISCV_IOMMU_QUEUE_ENABLE BIT(0)
|
||||
#define RISCV_IOMMU_QUEUE_INTR_ENABLE BIT(1)
|
||||
#define RISCV_IOMMU_QUEUE_MEM_FAULT BIT(8)
|
||||
#define RISCV_IOMMU_QUEUE_OVERFLOW BIT(9)
|
||||
#define RISCV_IOMMU_QUEUE_ACTIVE BIT(16)
|
||||
#define RISCV_IOMMU_QUEUE_BUSY BIT(17)
|
||||
|
||||
#define RISCV_IOMMU_ATP_PPN_FIELD GENMASK_ULL(43, 0)
|
||||
#define RISCV_IOMMU_ATP_MODE_FIELD GENMASK_ULL(63, 60)
|
||||
|
||||
/* 5.3 IOMMU Capabilities (64bits) */
|
||||
#define RISCV_IOMMU_REG_CAPABILITIES 0x0000
|
||||
#define RISCV_IOMMU_CAPABILITIES_VERSION GENMASK_ULL(7, 0)
|
||||
#define RISCV_IOMMU_CAPABILITIES_SV32 BIT_ULL(8)
|
||||
#define RISCV_IOMMU_CAPABILITIES_SV39 BIT_ULL(9)
|
||||
#define RISCV_IOMMU_CAPABILITIES_SV48 BIT_ULL(10)
|
||||
#define RISCV_IOMMU_CAPABILITIES_SV57 BIT_ULL(11)
|
||||
#define RISCV_IOMMU_CAPABILITIES_SVPBMT BIT_ULL(15)
|
||||
#define RISCV_IOMMU_CAPABILITIES_SV32X4 BIT_ULL(16)
|
||||
#define RISCV_IOMMU_CAPABILITIES_SV39X4 BIT_ULL(17)
|
||||
#define RISCV_IOMMU_CAPABILITIES_SV48X4 BIT_ULL(18)
|
||||
#define RISCV_IOMMU_CAPABILITIES_SV57X4 BIT_ULL(19)
|
||||
#define RISCV_IOMMU_CAPABILITIES_AMO_MRIF BIT_ULL(21)
|
||||
#define RISCV_IOMMU_CAPABILITIES_MSI_FLAT BIT_ULL(22)
|
||||
#define RISCV_IOMMU_CAPABILITIES_MSI_MRIF BIT_ULL(23)
|
||||
#define RISCV_IOMMU_CAPABILITIES_AMO_HWAD BIT_ULL(24)
|
||||
#define RISCV_IOMMU_CAPABILITIES_ATS BIT_ULL(25)
|
||||
#define RISCV_IOMMU_CAPABILITIES_T2GPA BIT_ULL(26)
|
||||
#define RISCV_IOMMU_CAPABILITIES_END BIT_ULL(27)
|
||||
#define RISCV_IOMMU_CAPABILITIES_IGS GENMASK_ULL(29, 28)
|
||||
#define RISCV_IOMMU_CAPABILITIES_HPM BIT_ULL(30)
|
||||
#define RISCV_IOMMU_CAPABILITIES_DBG BIT_ULL(31)
|
||||
#define RISCV_IOMMU_CAPABILITIES_PAS GENMASK_ULL(37, 32)
|
||||
#define RISCV_IOMMU_CAPABILITIES_PD8 BIT_ULL(38)
|
||||
#define RISCV_IOMMU_CAPABILITIES_PD17 BIT_ULL(39)
|
||||
#define RISCV_IOMMU_CAPABILITIES_PD20 BIT_ULL(40)
|
||||
|
||||
/**
|
||||
* enum riscv_iommu_igs_settings - Interrupt Generation Support Settings
|
||||
* @RISCV_IOMMU_CAPABILITIES_IGS_MSI: IOMMU supports only MSI generation
|
||||
* @RISCV_IOMMU_CAPABILITIES_IGS_WSI: IOMMU supports only Wired-Signaled interrupt
|
||||
* @RISCV_IOMMU_CAPABILITIES_IGS_BOTH: IOMMU supports both MSI and WSI generation
|
||||
* @RISCV_IOMMU_CAPABILITIES_IGS_RSRV: Reserved for standard use
|
||||
*/
|
||||
enum riscv_iommu_igs_settings {
|
||||
RISCV_IOMMU_CAPABILITIES_IGS_MSI = 0,
|
||||
RISCV_IOMMU_CAPABILITIES_IGS_WSI = 1,
|
||||
RISCV_IOMMU_CAPABILITIES_IGS_BOTH = 2,
|
||||
RISCV_IOMMU_CAPABILITIES_IGS_RSRV = 3
|
||||
};
|
||||
|
||||
/* 5.4 Features control register (32bits) */
|
||||
#define RISCV_IOMMU_REG_FCTL 0x0008
|
||||
#define RISCV_IOMMU_FCTL_BE BIT(0)
|
||||
#define RISCV_IOMMU_FCTL_WSI BIT(1)
|
||||
#define RISCV_IOMMU_FCTL_GXL BIT(2)
|
||||
|
||||
/* 5.5 Device-directory-table pointer (64bits) */
|
||||
#define RISCV_IOMMU_REG_DDTP 0x0010
|
||||
#define RISCV_IOMMU_DDTP_IOMMU_MODE GENMASK_ULL(3, 0)
|
||||
#define RISCV_IOMMU_DDTP_BUSY BIT_ULL(4)
|
||||
#define RISCV_IOMMU_DDTP_PPN RISCV_IOMMU_PPN_FIELD
|
||||
|
||||
/**
|
||||
* enum riscv_iommu_ddtp_modes - IOMMU translation modes
|
||||
* @RISCV_IOMMU_DDTP_IOMMU_MODE_OFF: No inbound transactions allowed
|
||||
* @RISCV_IOMMU_DDTP_IOMMU_MODE_BARE: Pass-through mode
|
||||
* @RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL: One-level DDT
|
||||
* @RISCV_IOMMU_DDTP_IOMMU_MODE_2LVL: Two-level DDT
|
||||
* @RISCV_IOMMU_DDTP_IOMMU_MODE_3LVL: Three-level DDT
|
||||
* @RISCV_IOMMU_DDTP_IOMMU_MODE_MAX: Max value allowed by specification
|
||||
*/
|
||||
enum riscv_iommu_ddtp_modes {
|
||||
RISCV_IOMMU_DDTP_IOMMU_MODE_OFF = 0,
|
||||
RISCV_IOMMU_DDTP_IOMMU_MODE_BARE = 1,
|
||||
RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL = 2,
|
||||
RISCV_IOMMU_DDTP_IOMMU_MODE_2LVL = 3,
|
||||
RISCV_IOMMU_DDTP_IOMMU_MODE_3LVL = 4,
|
||||
RISCV_IOMMU_DDTP_IOMMU_MODE_MAX = 4
|
||||
};
|
||||
|
||||
/* 5.6 Command Queue Base (64bits) */
|
||||
#define RISCV_IOMMU_REG_CQB 0x0018
|
||||
#define RISCV_IOMMU_CQB_ENTRIES RISCV_IOMMU_QUEUE_LOG2SZ_FIELD
|
||||
#define RISCV_IOMMU_CQB_PPN RISCV_IOMMU_PPN_FIELD
|
||||
|
||||
/* 5.7 Command Queue head (32bits) */
|
||||
#define RISCV_IOMMU_REG_CQH 0x0020
|
||||
#define RISCV_IOMMU_CQH_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||
|
||||
/* 5.8 Command Queue tail (32bits) */
|
||||
#define RISCV_IOMMU_REG_CQT 0x0024
|
||||
#define RISCV_IOMMU_CQT_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||
|
||||
/* 5.9 Fault Queue Base (64bits) */
|
||||
#define RISCV_IOMMU_REG_FQB 0x0028
|
||||
#define RISCV_IOMMU_FQB_ENTRIES RISCV_IOMMU_QUEUE_LOG2SZ_FIELD
|
||||
#define RISCV_IOMMU_FQB_PPN RISCV_IOMMU_PPN_FIELD
|
||||
|
||||
/* 5.10 Fault Queue Head (32bits) */
|
||||
#define RISCV_IOMMU_REG_FQH 0x0030
|
||||
#define RISCV_IOMMU_FQH_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||
|
||||
/* 5.11 Fault Queue tail (32bits) */
|
||||
#define RISCV_IOMMU_REG_FQT 0x0034
|
||||
#define RISCV_IOMMU_FQT_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||
|
||||
/* 5.12 Page Request Queue base (64bits) */
|
||||
#define RISCV_IOMMU_REG_PQB 0x0038
|
||||
#define RISCV_IOMMU_PQB_ENTRIES RISCV_IOMMU_QUEUE_LOG2SZ_FIELD
|
||||
#define RISCV_IOMMU_PQB_PPN RISCV_IOMMU_PPN_FIELD
|
||||
|
||||
/* 5.13 Page Request Queue head (32bits) */
|
||||
#define RISCV_IOMMU_REG_PQH 0x0040
|
||||
#define RISCV_IOMMU_PQH_INDEX RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||
|
||||
/* 5.14 Page Request Queue tail (32bits) */
|
||||
#define RISCV_IOMMU_REG_PQT 0x0044
|
||||
#define RISCV_IOMMU_PQT_INDEX_MASK RISCV_IOMMU_QUEUE_INDEX_FIELD
|
||||
|
||||
/* 5.15 Command Queue CSR (32bits) */
|
||||
#define RISCV_IOMMU_REG_CQCSR 0x0048
|
||||
#define RISCV_IOMMU_CQCSR_CQEN RISCV_IOMMU_QUEUE_ENABLE
|
||||
#define RISCV_IOMMU_CQCSR_CIE RISCV_IOMMU_QUEUE_INTR_ENABLE
|
||||
#define RISCV_IOMMU_CQCSR_CQMF RISCV_IOMMU_QUEUE_MEM_FAULT
|
||||
#define RISCV_IOMMU_CQCSR_CMD_TO BIT(9)
|
||||
#define RISCV_IOMMU_CQCSR_CMD_ILL BIT(10)
|
||||
#define RISCV_IOMMU_CQCSR_FENCE_W_IP BIT(11)
|
||||
#define RISCV_IOMMU_CQCSR_CQON RISCV_IOMMU_QUEUE_ACTIVE
|
||||
#define RISCV_IOMMU_CQCSR_BUSY RISCV_IOMMU_QUEUE_BUSY
|
||||
|
||||
/* 5.16 Fault Queue CSR (32bits) */
|
||||
#define RISCV_IOMMU_REG_FQCSR 0x004C
|
||||
#define RISCV_IOMMU_FQCSR_FQEN RISCV_IOMMU_QUEUE_ENABLE
|
||||
#define RISCV_IOMMU_FQCSR_FIE RISCV_IOMMU_QUEUE_INTR_ENABLE
|
||||
#define RISCV_IOMMU_FQCSR_FQMF RISCV_IOMMU_QUEUE_MEM_FAULT
|
||||
#define RISCV_IOMMU_FQCSR_FQOF RISCV_IOMMU_QUEUE_OVERFLOW
|
||||
#define RISCV_IOMMU_FQCSR_FQON RISCV_IOMMU_QUEUE_ACTIVE
|
||||
#define RISCV_IOMMU_FQCSR_BUSY RISCV_IOMMU_QUEUE_BUSY
|
||||
|
||||
/* 5.17 Page Request Queue CSR (32bits) */
|
||||
#define RISCV_IOMMU_REG_PQCSR 0x0050
|
||||
#define RISCV_IOMMU_PQCSR_PQEN RISCV_IOMMU_QUEUE_ENABLE
|
||||
#define RISCV_IOMMU_PQCSR_PIE RISCV_IOMMU_QUEUE_INTR_ENABLE
|
||||
#define RISCV_IOMMU_PQCSR_PQMF RISCV_IOMMU_QUEUE_MEM_FAULT
|
||||
#define RISCV_IOMMU_PQCSR_PQOF RISCV_IOMMU_QUEUE_OVERFLOW
|
||||
#define RISCV_IOMMU_PQCSR_PQON RISCV_IOMMU_QUEUE_ACTIVE
|
||||
#define RISCV_IOMMU_PQCSR_BUSY RISCV_IOMMU_QUEUE_BUSY
|
||||
|
||||
/* 5.18 Interrupt Pending Status (32bits) */
|
||||
#define RISCV_IOMMU_REG_IPSR 0x0054
|
||||
|
||||
#define RISCV_IOMMU_INTR_CQ 0
|
||||
#define RISCV_IOMMU_INTR_FQ 1
|
||||
#define RISCV_IOMMU_INTR_PM 2
|
||||
#define RISCV_IOMMU_INTR_PQ 3
|
||||
#define RISCV_IOMMU_INTR_COUNT 4
|
||||
|
||||
#define RISCV_IOMMU_IPSR_CIP BIT(RISCV_IOMMU_INTR_CQ)
|
||||
#define RISCV_IOMMU_IPSR_FIP BIT(RISCV_IOMMU_INTR_FQ)
|
||||
#define RISCV_IOMMU_IPSR_PMIP BIT(RISCV_IOMMU_INTR_PM)
|
||||
#define RISCV_IOMMU_IPSR_PIP BIT(RISCV_IOMMU_INTR_PQ)
|
||||
|
||||
/* 5.19 Performance monitoring counter overflow status (32bits) */
|
||||
#define RISCV_IOMMU_REG_IOCOUNTOVF 0x0058
|
||||
#define RISCV_IOMMU_IOCOUNTOVF_CY BIT(0)
|
||||
#define RISCV_IOMMU_IOCOUNTOVF_HPM GENMASK_ULL(31, 1)
|
||||
|
||||
/* 5.20 Performance monitoring counter inhibits (32bits) */
|
||||
#define RISCV_IOMMU_REG_IOCOUNTINH 0x005C
|
||||
#define RISCV_IOMMU_IOCOUNTINH_CY BIT(0)
|
||||
#define RISCV_IOMMU_IOCOUNTINH_HPM GENMASK(31, 1)
|
||||
|
||||
/* 5.21 Performance monitoring cycles counter (64bits) */
|
||||
#define RISCV_IOMMU_REG_IOHPMCYCLES 0x0060
|
||||
#define RISCV_IOMMU_IOHPMCYCLES_COUNTER GENMASK_ULL(62, 0)
|
||||
#define RISCV_IOMMU_IOHPMCYCLES_OF BIT_ULL(63)
|
||||
|
||||
/* 5.22 Performance monitoring event counters (31 * 64bits) */
|
||||
#define RISCV_IOMMU_REG_IOHPMCTR_BASE 0x0068
|
||||
#define RISCV_IOMMU_REG_IOHPMCTR(_n) (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
|
||||
|
||||
/* 5.23 Performance monitoring event selectors (31 * 64bits) */
|
||||
#define RISCV_IOMMU_REG_IOHPMEVT_BASE 0x0160
|
||||
#define RISCV_IOMMU_REG_IOHPMEVT(_n) (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
|
||||
#define RISCV_IOMMU_IOHPMEVT_EVENTID GENMASK_ULL(14, 0)
|
||||
#define RISCV_IOMMU_IOHPMEVT_DMASK BIT_ULL(15)
|
||||
#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
|
||||
#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
|
||||
#define RISCV_IOMMU_IOHPMEVT_PV_PSCV BIT_ULL(60)
|
||||
#define RISCV_IOMMU_IOHPMEVT_DV_GSCV BIT_ULL(61)
|
||||
#define RISCV_IOMMU_IOHPMEVT_IDT BIT_ULL(62)
|
||||
#define RISCV_IOMMU_IOHPMEVT_OF BIT_ULL(63)
|
||||
|
||||
/* Number of defined performance-monitoring event selectors */
|
||||
#define RISCV_IOMMU_IOHPMEVT_CNT 31
|
||||
|
||||
/**
|
||||
* enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
|
||||
*
|
||||
* @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
|
||||
* @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
|
||||
* @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
|
||||
* @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
|
||||
* @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
|
||||
* @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
|
||||
* @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
|
||||
* @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
|
||||
* @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
|
||||
* @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
|
||||
*/
|
||||
enum riscv_iommu_hpmevent_id {
|
||||
RISCV_IOMMU_HPMEVENT_INVALID = 0,
|
||||
RISCV_IOMMU_HPMEVENT_URQ = 1,
|
||||
RISCV_IOMMU_HPMEVENT_TRQ = 2,
|
||||
RISCV_IOMMU_HPMEVENT_ATS_RQ = 3,
|
||||
RISCV_IOMMU_HPMEVENT_TLB_MISS = 4,
|
||||
RISCV_IOMMU_HPMEVENT_DD_WALK = 5,
|
||||
RISCV_IOMMU_HPMEVENT_PD_WALK = 6,
|
||||
RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
|
||||
RISCV_IOMMU_HPMEVENT_G_WALKS = 8,
|
||||
RISCV_IOMMU_HPMEVENT_MAX = 9
|
||||
};
|
||||
|
||||
/* 5.24 Translation request IOVA (64bits) */
|
||||
#define RISCV_IOMMU_REG_TR_REQ_IOVA 0x0258
|
||||
#define RISCV_IOMMU_TR_REQ_IOVA_VPN GENMASK_ULL(63, 12)
|
||||
|
||||
/* 5.25 Translation request control (64bits) */
|
||||
#define RISCV_IOMMU_REG_TR_REQ_CTL 0x0260
|
||||
#define RISCV_IOMMU_TR_REQ_CTL_GO_BUSY BIT_ULL(0)
|
||||
#define RISCV_IOMMU_TR_REQ_CTL_PRIV BIT_ULL(1)
|
||||
#define RISCV_IOMMU_TR_REQ_CTL_EXE BIT_ULL(2)
|
||||
#define RISCV_IOMMU_TR_REQ_CTL_NW BIT_ULL(3)
|
||||
#define RISCV_IOMMU_TR_REQ_CTL_PID GENMASK_ULL(31, 12)
|
||||
#define RISCV_IOMMU_TR_REQ_CTL_PV BIT_ULL(32)
|
||||
#define RISCV_IOMMU_TR_REQ_CTL_DID GENMASK_ULL(63, 40)
|
||||
|
||||
/* 5.26 Translation request response (64bits) */
|
||||
#define RISCV_IOMMU_REG_TR_RESPONSE 0x0268
|
||||
#define RISCV_IOMMU_TR_RESPONSE_FAULT BIT_ULL(0)
|
||||
#define RISCV_IOMMU_TR_RESPONSE_PBMT GENMASK_ULL(8, 7)
|
||||
#define RISCV_IOMMU_TR_RESPONSE_SZ BIT_ULL(9)
|
||||
#define RISCV_IOMMU_TR_RESPONSE_PPN RISCV_IOMMU_PPN_FIELD
|
||||
|
||||
/* 5.27 Interrupt cause to vector (64bits) */
|
||||
#define RISCV_IOMMU_REG_ICVEC 0x02F8
|
||||
#define RISCV_IOMMU_ICVEC_CIV GENMASK_ULL(3, 0)
|
||||
#define RISCV_IOMMU_ICVEC_FIV GENMASK_ULL(7, 4)
|
||||
#define RISCV_IOMMU_ICVEC_PMIV GENMASK_ULL(11, 8)
|
||||
#define RISCV_IOMMU_ICVEC_PIV GENMASK_ULL(15, 12)
|
||||
|
||||
/* 5.28 MSI Configuration table (32 * 64bits) */
|
||||
#define RISCV_IOMMU_REG_MSI_CFG_TBL 0x0300
|
||||
#define RISCV_IOMMU_REG_MSI_CFG_TBL_ADDR(_n) \
|
||||
(RISCV_IOMMU_REG_MSI_CFG_TBL + ((_n) * 0x10))
|
||||
#define RISCV_IOMMU_MSI_CFG_TBL_ADDR GENMASK_ULL(55, 2)
|
||||
#define RISCV_IOMMU_REG_MSI_CFG_TBL_DATA(_n) \
|
||||
(RISCV_IOMMU_REG_MSI_CFG_TBL + ((_n) * 0x10) + 0x08)
|
||||
#define RISCV_IOMMU_MSI_CFG_TBL_DATA GENMASK_ULL(31, 0)
|
||||
#define RISCV_IOMMU_REG_MSI_CFG_TBL_CTRL(_n) \
|
||||
(RISCV_IOMMU_REG_MSI_CFG_TBL + ((_n) * 0x10) + 0x0C)
|
||||
#define RISCV_IOMMU_MSI_CFG_TBL_CTRL_M BIT_ULL(0)
|
||||
|
||||
#define RISCV_IOMMU_REG_SIZE 0x1000
|
||||
|
||||
/*
|
||||
* Chapter 2: Data structures
|
||||
*/
|
||||
|
||||
/*
|
||||
* Device Directory Table macros for non-leaf nodes
|
||||
*/
|
||||
#define RISCV_IOMMU_DDTE_V BIT_ULL(0)
|
||||
#define RISCV_IOMMU_DDTE_PPN RISCV_IOMMU_PPN_FIELD
|
||||
|
||||
/**
|
||||
* struct riscv_iommu_dc - Device Context
|
||||
* @tc: Translation Control
|
||||
* @iohgatp: I/O Hypervisor guest address translation and protection
|
||||
* (Second stage context)
|
||||
* @ta: Translation Attributes
|
||||
* @fsc: First stage context
|
||||
* @msiptp: MSI page table pointer
|
||||
* @msi_addr_mask: MSI address mask
|
||||
* @msi_addr_pattern: MSI address pattern
|
||||
* @_reserved: Reserved for future use, padding
|
||||
*
|
||||
* This structure is used for leaf nodes on the Device Directory Table,
|
||||
* in case RISCV_IOMMU_CAPABILITIES_MSI_FLAT is not set, the bottom 4 fields
|
||||
* are not present and are skipped with pointer arithmetic to avoid
|
||||
* casting, check out riscv_iommu_get_dc().
|
||||
* See section 2.1 for more details
|
||||
*/
|
||||
struct riscv_iommu_dc {
|
||||
u64 tc;
|
||||
u64 iohgatp;
|
||||
u64 ta;
|
||||
u64 fsc;
|
||||
u64 msiptp;
|
||||
u64 msi_addr_mask;
|
||||
u64 msi_addr_pattern;
|
||||
u64 _reserved;
|
||||
};
|
||||
|
||||
/* Translation control fields */
|
||||
#define RISCV_IOMMU_DC_TC_V BIT_ULL(0)
|
||||
#define RISCV_IOMMU_DC_TC_EN_ATS BIT_ULL(1)
|
||||
#define RISCV_IOMMU_DC_TC_EN_PRI BIT_ULL(2)
|
||||
#define RISCV_IOMMU_DC_TC_T2GPA BIT_ULL(3)
|
||||
#define RISCV_IOMMU_DC_TC_DTF BIT_ULL(4)
|
||||
#define RISCV_IOMMU_DC_TC_PDTV BIT_ULL(5)
|
||||
#define RISCV_IOMMU_DC_TC_PRPR BIT_ULL(6)
|
||||
#define RISCV_IOMMU_DC_TC_GADE BIT_ULL(7)
|
||||
#define RISCV_IOMMU_DC_TC_SADE BIT_ULL(8)
|
||||
#define RISCV_IOMMU_DC_TC_DPE BIT_ULL(9)
|
||||
#define RISCV_IOMMU_DC_TC_SBE BIT_ULL(10)
|
||||
#define RISCV_IOMMU_DC_TC_SXL BIT_ULL(11)
|
||||
|
||||
/* Second-stage (aka G-stage) context fields */
|
||||
#define RISCV_IOMMU_DC_IOHGATP_PPN RISCV_IOMMU_ATP_PPN_FIELD
|
||||
#define RISCV_IOMMU_DC_IOHGATP_GSCID GENMASK_ULL(59, 44)
|
||||
#define RISCV_IOMMU_DC_IOHGATP_MODE RISCV_IOMMU_ATP_MODE_FIELD
|
||||
|
||||
/**
|
||||
* enum riscv_iommu_dc_iohgatp_modes - Guest address translation/protection modes
|
||||
* @RISCV_IOMMU_DC_IOHGATP_MODE_BARE: No translation/protection
|
||||
* @RISCV_IOMMU_DC_IOHGATP_MODE_SV32X4: Sv32x4 (2-bit extension of Sv32), when fctl.GXL == 1
|
||||
* @RISCV_IOMMU_DC_IOHGATP_MODE_SV39X4: Sv39x4 (2-bit extension of Sv39), when fctl.GXL == 0
|
||||
* @RISCV_IOMMU_DC_IOHGATP_MODE_SV48X4: Sv48x4 (2-bit extension of Sv48), when fctl.GXL == 0
|
||||
* @RISCV_IOMMU_DC_IOHGATP_MODE_SV57X4: Sv57x4 (2-bit extension of Sv57), when fctl.GXL == 0
|
||||
*/
|
||||
enum riscv_iommu_dc_iohgatp_modes {
|
||||
RISCV_IOMMU_DC_IOHGATP_MODE_BARE = 0,
|
||||
RISCV_IOMMU_DC_IOHGATP_MODE_SV32X4 = 8,
|
||||
RISCV_IOMMU_DC_IOHGATP_MODE_SV39X4 = 8,
|
||||
RISCV_IOMMU_DC_IOHGATP_MODE_SV48X4 = 9,
|
||||
RISCV_IOMMU_DC_IOHGATP_MODE_SV57X4 = 10
|
||||
};
|
||||
|
||||
/* Translation attributes fields */
|
||||
#define RISCV_IOMMU_DC_TA_PSCID GENMASK_ULL(31, 12)
|
||||
|
||||
/* First-stage context fields */
|
||||
#define RISCV_IOMMU_DC_FSC_PPN RISCV_IOMMU_ATP_PPN_FIELD
|
||||
#define RISCV_IOMMU_DC_FSC_MODE RISCV_IOMMU_ATP_MODE_FIELD
|
||||
|
||||
/**
|
||||
* enum riscv_iommu_dc_fsc_atp_modes - First stage address translation/protection modes
|
||||
* @RISCV_IOMMU_DC_FSC_MODE_BARE: No translation/protection
|
||||
* @RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV32: Sv32, when dc.tc.SXL == 1
|
||||
* @RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39: Sv39, when dc.tc.SXL == 0
|
||||
* @RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48: Sv48, when dc.tc.SXL == 0
|
||||
* @RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57: Sv57, when dc.tc.SXL == 0
|
||||
* @RISCV_IOMMU_DC_FSC_PDTP_MODE_PD8: 1lvl PDT, 8bit process ids
|
||||
* @RISCV_IOMMU_DC_FSC_PDTP_MODE_PD17: 2lvl PDT, 17bit process ids
|
||||
* @RISCV_IOMMU_DC_FSC_PDTP_MODE_PD20: 3lvl PDT, 20bit process ids
|
||||
*
|
||||
* FSC holds IOSATP when RISCV_IOMMU_DC_TC_PDTV is 0 and PDTP otherwise.
|
||||
* IOSATP controls the first stage address translation (same as the satp register on
|
||||
* the RISC-V MMU), and PDTP holds the process directory table, used to select a
|
||||
* first stage page table based on a process id (for devices that support multiple
|
||||
* process ids).
|
||||
*/
|
||||
enum riscv_iommu_dc_fsc_atp_modes {
|
||||
RISCV_IOMMU_DC_FSC_MODE_BARE = 0,
|
||||
RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV32 = 8,
|
||||
RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39 = 8,
|
||||
RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48 = 9,
|
||||
RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57 = 10,
|
||||
RISCV_IOMMU_DC_FSC_PDTP_MODE_PD8 = 1,
|
||||
RISCV_IOMMU_DC_FSC_PDTP_MODE_PD17 = 2,
|
||||
RISCV_IOMMU_DC_FSC_PDTP_MODE_PD20 = 3
|
||||
};
|
||||
|
||||
/* MSI page table pointer */
|
||||
#define RISCV_IOMMU_DC_MSIPTP_PPN RISCV_IOMMU_ATP_PPN_FIELD
|
||||
#define RISCV_IOMMU_DC_MSIPTP_MODE RISCV_IOMMU_ATP_MODE_FIELD
|
||||
#define RISCV_IOMMU_DC_MSIPTP_MODE_OFF 0
|
||||
#define RISCV_IOMMU_DC_MSIPTP_MODE_FLAT 1
|
||||
|
||||
/* MSI address mask */
|
||||
#define RISCV_IOMMU_DC_MSI_ADDR_MASK GENMASK_ULL(51, 0)
|
||||
|
||||
/* MSI address pattern */
|
||||
#define RISCV_IOMMU_DC_MSI_PATTERN GENMASK_ULL(51, 0)
|
||||
|
||||
/**
|
||||
* struct riscv_iommu_pc - Process Context
|
||||
* @ta: Translation Attributes
|
||||
* @fsc: First stage context
|
||||
*
|
||||
* This structure is used for leaf nodes on the Process Directory Table
|
||||
* See section 2.3 for more details
|
||||
*/
|
||||
struct riscv_iommu_pc {
|
||||
u64 ta;
|
||||
u64 fsc;
|
||||
};
|
||||
|
||||
/* Translation attributes fields */
|
||||
#define RISCV_IOMMU_PC_TA_V BIT_ULL(0)
|
||||
#define RISCV_IOMMU_PC_TA_ENS BIT_ULL(1)
|
||||
#define RISCV_IOMMU_PC_TA_SUM BIT_ULL(2)
|
||||
#define RISCV_IOMMU_PC_TA_PSCID GENMASK_ULL(31, 12)
|
||||
|
||||
/* First stage context fields */
|
||||
#define RISCV_IOMMU_PC_FSC_PPN RISCV_IOMMU_ATP_PPN_FIELD
|
||||
#define RISCV_IOMMU_PC_FSC_MODE RISCV_IOMMU_ATP_MODE_FIELD
|
||||
|
||||
/*
|
||||
* Chapter 3: In-memory queue interface
|
||||
*/
|
||||
|
||||
/**
|
||||
* struct riscv_iommu_command - Generic IOMMU command structure
|
||||
* @dword0: Includes the opcode and the function identifier
|
||||
* @dword1: Opcode specific data
|
||||
*
|
||||
* The commands are interpreted as two 64bit fields, where the first
|
||||
* 7bits of the first field are the opcode which also defines the
|
||||
* command's format, followed by a 3bit field that specifies the
|
||||
* function invoked by that command, and the rest is opcode-specific.
|
||||
* This is a generic struct which will be populated differently
|
||||
* according to each command. For more infos on the commands and
|
||||
* the command queue check section 3.1.
|
||||
*/
|
||||
struct riscv_iommu_command {
|
||||
u64 dword0;
|
||||
u64 dword1;
|
||||
};
|
||||
|
||||
/* Fields on dword0, common for all commands */
|
||||
#define RISCV_IOMMU_CMD_OPCODE GENMASK_ULL(6, 0)
|
||||
#define RISCV_IOMMU_CMD_FUNC GENMASK_ULL(9, 7)
|
||||
|
||||
/* 3.1.1 IOMMU Page-table cache invalidation */
|
||||
/* Fields on dword0 */
|
||||
#define RISCV_IOMMU_CMD_IOTINVAL_OPCODE 1
|
||||
#define RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA 0
|
||||
#define RISCV_IOMMU_CMD_IOTINVAL_FUNC_GVMA 1
|
||||
#define RISCV_IOMMU_CMD_IOTINVAL_AV BIT_ULL(10)
|
||||
#define RISCV_IOMMU_CMD_IOTINVAL_PSCID GENMASK_ULL(31, 12)
|
||||
#define RISCV_IOMMU_CMD_IOTINVAL_PSCV BIT_ULL(32)
|
||||
#define RISCV_IOMMU_CMD_IOTINVAL_GV BIT_ULL(33)
|
||||
#define RISCV_IOMMU_CMD_IOTINVAL_GSCID GENMASK_ULL(59, 44)
|
||||
/* dword1[61:10] is the 4K-aligned page address */
|
||||
#define RISCV_IOMMU_CMD_IOTINVAL_ADDR GENMASK_ULL(61, 10)
|
||||
|
||||
/* 3.1.2 IOMMU Command Queue Fences */
|
||||
/* Fields on dword0 */
|
||||
#define RISCV_IOMMU_CMD_IOFENCE_OPCODE 2
|
||||
#define RISCV_IOMMU_CMD_IOFENCE_FUNC_C 0
|
||||
#define RISCV_IOMMU_CMD_IOFENCE_AV BIT_ULL(10)
|
||||
#define RISCV_IOMMU_CMD_IOFENCE_WSI BIT_ULL(11)
|
||||
#define RISCV_IOMMU_CMD_IOFENCE_PR BIT_ULL(12)
|
||||
#define RISCV_IOMMU_CMD_IOFENCE_PW BIT_ULL(13)
|
||||
#define RISCV_IOMMU_CMD_IOFENCE_DATA GENMASK_ULL(63, 32)
|
||||
/* dword1 is the address, word-size aligned and shifted to the right by two bits. */
|
||||
|
||||
/* 3.1.3 IOMMU Directory cache invalidation */
|
||||
/* Fields on dword0 */
|
||||
#define RISCV_IOMMU_CMD_IODIR_OPCODE 3
|
||||
#define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT 0
|
||||
#define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT 1
|
||||
#define RISCV_IOMMU_CMD_IODIR_PID GENMASK_ULL(31, 12)
|
||||
#define RISCV_IOMMU_CMD_IODIR_DV BIT_ULL(33)
|
||||
#define RISCV_IOMMU_CMD_IODIR_DID GENMASK_ULL(63, 40)
|
||||
/* dword1 is reserved for standard use */
|
||||
|
||||
/* 3.1.4 IOMMU PCIe ATS */
|
||||
/* Fields on dword0 */
|
||||
#define RISCV_IOMMU_CMD_ATS_OPCODE 4
|
||||
#define RISCV_IOMMU_CMD_ATS_FUNC_INVAL 0
|
||||
#define RISCV_IOMMU_CMD_ATS_FUNC_PRGR 1
|
||||
#define RISCV_IOMMU_CMD_ATS_PID GENMASK_ULL(31, 12)
|
||||
#define RISCV_IOMMU_CMD_ATS_PV BIT_ULL(32)
|
||||
#define RISCV_IOMMU_CMD_ATS_DSV BIT_ULL(33)
|
||||
#define RISCV_IOMMU_CMD_ATS_RID GENMASK_ULL(55, 40)
|
||||
#define RISCV_IOMMU_CMD_ATS_DSEG GENMASK_ULL(63, 56)
|
||||
/* dword1 is the ATS payload, two different payload types for INVAL and PRGR */
|
||||
|
||||
/* ATS.INVAL payload*/
|
||||
#define RISCV_IOMMU_CMD_ATS_INVAL_G BIT_ULL(0)
|
||||
/* Bits 1 - 10 are zeroed */
|
||||
#define RISCV_IOMMU_CMD_ATS_INVAL_S BIT_ULL(11)
|
||||
#define RISCV_IOMMU_CMD_ATS_INVAL_UADDR GENMASK_ULL(63, 12)
|
||||
|
||||
/* ATS.PRGR payload */
|
||||
/* Bits 0 - 31 are zeroed */
|
||||
#define RISCV_IOMMU_CMD_ATS_PRGR_PRG_INDEX GENMASK_ULL(40, 32)
|
||||
/* Bits 41 - 43 are zeroed */
|
||||
#define RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE GENMASK_ULL(47, 44)
|
||||
#define RISCV_IOMMU_CMD_ATS_PRGR_DST_ID GENMASK_ULL(63, 48)
|
||||
|
||||
/**
|
||||
* struct riscv_iommu_fq_record - Fault/Event Queue Record
|
||||
* @hdr: Header, includes fault/event cause, PID/DID, transaction type etc
|
||||
* @_reserved: Low 32bits for custom use, high 32bits for standard use
|
||||
* @iotval: Transaction-type/cause specific format
|
||||
* @iotval2: Cause specific format
|
||||
*
|
||||
* The fault/event queue reports events and failures raised when
|
||||
* processing transactions. Each record is a 32byte structure where
|
||||
* the first dword has a fixed format for providing generic infos
|
||||
* regarding the fault/event, and two more dwords are there for
|
||||
* fault/event-specific information. For more details see section
|
||||
* 3.2.
|
||||
*/
|
||||
struct riscv_iommu_fq_record {
|
||||
u64 hdr;
|
||||
u64 _reserved;
|
||||
u64 iotval;
|
||||
u64 iotval2;
|
||||
};
|
||||
|
||||
/* Fields on header */
|
||||
#define RISCV_IOMMU_FQ_HDR_CAUSE GENMASK_ULL(11, 0)
|
||||
#define RISCV_IOMMU_FQ_HDR_PID GENMASK_ULL(31, 12)
|
||||
#define RISCV_IOMMU_FQ_HDR_PV BIT_ULL(32)
|
||||
#define RISCV_IOMMU_FQ_HDR_PRIV BIT_ULL(33)
|
||||
#define RISCV_IOMMU_FQ_HDR_TTYP GENMASK_ULL(39, 34)
|
||||
#define RISCV_IOMMU_FQ_HDR_DID GENMASK_ULL(63, 40)
|
||||
|
||||
/**
|
||||
* enum riscv_iommu_fq_causes - Fault/event cause values
|
||||
* @RISCV_IOMMU_FQ_CAUSE_INST_FAULT: Instruction access fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_RD_ADDR_MISALIGNED: Read address misaligned
|
||||
* @RISCV_IOMMU_FQ_CAUSE_RD_FAULT: Read load fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_WR_ADDR_MISALIGNED: Write/AMO address misaligned
|
||||
* @RISCV_IOMMU_FQ_CAUSE_WR_FAULT: Write/AMO access fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_INST_FAULT_S: Instruction page fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S: Read page fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S: Write/AMO page fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_INST_FAULT_VS: Instruction guest page fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS: Read guest page fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS: Write/AMO guest page fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED: All inbound transactions disallowed
|
||||
* @RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT: DDT entry load access fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_DDT_INVALID: DDT entry invalid
|
||||
* @RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED: DDT entry misconfigured
|
||||
* @RISCV_IOMMU_FQ_CAUSE_TTYP_BLOCKED: Transaction type disallowed
|
||||
* @RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT: MSI PTE load access fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_MSI_INVALID: MSI PTE invalid
|
||||
* @RISCV_IOMMU_FQ_CAUSE_MSI_MISCONFIGURED: MSI PTE misconfigured
|
||||
* @RISCV_IOMMU_FQ_CAUSE_MRIF_FAULT: MRIF access fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT: PDT entry load access fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_PDT_INVALID: PDT entry invalid
|
||||
* @RISCV_IOMMU_FQ_CAUSE_PDT_MISCONFIGURED: PDT entry misconfigured
|
||||
* @RISCV_IOMMU_FQ_CAUSE_DDT_CORRUPTED: DDT data corruption
|
||||
* @RISCV_IOMMU_FQ_CAUSE_PDT_CORRUPTED: PDT data corruption
|
||||
* @RISCV_IOMMU_FQ_CAUSE_MSI_PT_CORRUPTED: MSI page table data corruption
|
||||
* @RISCV_IOMMU_FQ_CAUSE_MRIF_CORRUIPTED: MRIF data corruption
|
||||
* @RISCV_IOMMU_FQ_CAUSE_INTERNAL_DP_ERROR: Internal data path error
|
||||
* @RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT: IOMMU MSI write access fault
|
||||
* @RISCV_IOMMU_FQ_CAUSE_PT_CORRUPTED: First/second stage page table data corruption
|
||||
*
|
||||
* Values are on table 11 of the spec, encodings 275 - 2047 are reserved for standard
|
||||
* use, and 2048 - 4095 for custom use.
|
||||
*/
|
||||
enum riscv_iommu_fq_causes {
|
||||
RISCV_IOMMU_FQ_CAUSE_INST_FAULT = 1,
|
||||
RISCV_IOMMU_FQ_CAUSE_RD_ADDR_MISALIGNED = 4,
|
||||
RISCV_IOMMU_FQ_CAUSE_RD_FAULT = 5,
|
||||
RISCV_IOMMU_FQ_CAUSE_WR_ADDR_MISALIGNED = 6,
|
||||
RISCV_IOMMU_FQ_CAUSE_WR_FAULT = 7,
|
||||
RISCV_IOMMU_FQ_CAUSE_INST_FAULT_S = 12,
|
||||
RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S = 13,
|
||||
RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S = 15,
|
||||
RISCV_IOMMU_FQ_CAUSE_INST_FAULT_VS = 20,
|
||||
RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS = 21,
|
||||
RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS = 23,
|
||||
RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED = 256,
|
||||
RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT = 257,
|
||||
RISCV_IOMMU_FQ_CAUSE_DDT_INVALID = 258,
|
||||
RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED = 259,
|
||||
RISCV_IOMMU_FQ_CAUSE_TTYP_BLOCKED = 260,
|
||||
RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT = 261,
|
||||
RISCV_IOMMU_FQ_CAUSE_MSI_INVALID = 262,
|
||||
RISCV_IOMMU_FQ_CAUSE_MSI_MISCONFIGURED = 263,
|
||||
RISCV_IOMMU_FQ_CAUSE_MRIF_FAULT = 264,
|
||||
RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT = 265,
|
||||
RISCV_IOMMU_FQ_CAUSE_PDT_INVALID = 266,
|
||||
RISCV_IOMMU_FQ_CAUSE_PDT_MISCONFIGURED = 267,
|
||||
RISCV_IOMMU_FQ_CAUSE_DDT_CORRUPTED = 268,
|
||||
RISCV_IOMMU_FQ_CAUSE_PDT_CORRUPTED = 269,
|
||||
RISCV_IOMMU_FQ_CAUSE_MSI_PT_CORRUPTED = 270,
|
||||
RISCV_IOMMU_FQ_CAUSE_MRIF_CORRUIPTED = 271,
|
||||
RISCV_IOMMU_FQ_CAUSE_INTERNAL_DP_ERROR = 272,
|
||||
RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT = 273,
|
||||
RISCV_IOMMU_FQ_CAUSE_PT_CORRUPTED = 274
|
||||
};
|
||||
|
||||
/**
|
||||
* enum riscv_iommu_fq_ttypes: Fault/event transaction types
|
||||
* @RISCV_IOMMU_FQ_TTYP_NONE: None. Fault not caused by an inbound transaction.
|
||||
* @RISCV_IOMMU_FQ_TTYP_UADDR_INST_FETCH: Instruction fetch from untranslated address
|
||||
* @RISCV_IOMMU_FQ_TTYP_UADDR_RD: Read from untranslated address
|
||||
* @RISCV_IOMMU_FQ_TTYP_UADDR_WR: Write/AMO to untranslated address
|
||||
* @RISCV_IOMMU_FQ_TTYP_TADDR_INST_FETCH: Instruction fetch from translated address
|
||||
* @RISCV_IOMMU_FQ_TTYP_TADDR_RD: Read from translated address
|
||||
* @RISCV_IOMMU_FQ_TTYP_TADDR_WR: Write/AMO to translated address
|
||||
* @RISCV_IOMMU_FQ_TTYP_PCIE_ATS_REQ: PCIe ATS translation request
|
||||
* @RISCV_IOMMU_FQ_TTYP_PCIE_MSG_REQ: PCIe message request
|
||||
*
|
||||
* Values are on table 12 of the spec, type 4 and 10 - 31 are reserved for standard use
|
||||
* and 31 - 63 for custom use.
|
||||
*/
|
||||
enum riscv_iommu_fq_ttypes {
|
||||
RISCV_IOMMU_FQ_TTYP_NONE = 0,
|
||||
RISCV_IOMMU_FQ_TTYP_UADDR_INST_FETCH = 1,
|
||||
RISCV_IOMMU_FQ_TTYP_UADDR_RD = 2,
|
||||
RISCV_IOMMU_FQ_TTYP_UADDR_WR = 3,
|
||||
RISCV_IOMMU_FQ_TTYP_TADDR_INST_FETCH = 5,
|
||||
RISCV_IOMMU_FQ_TTYP_TADDR_RD = 6,
|
||||
RISCV_IOMMU_FQ_TTYP_TADDR_WR = 7,
|
||||
RISCV_IOMMU_FQ_TTYP_PCIE_ATS_REQ = 8,
|
||||
RISCV_IOMMU_FQ_TTYP_PCIE_MSG_REQ = 9,
|
||||
};
|
||||
|
||||
/**
|
||||
* struct riscv_iommu_pq_record - PCIe Page Request record
|
||||
* @hdr: Header, includes PID, DID etc
|
||||
* @payload: Holds the page address, request group and permission bits
|
||||
*
|
||||
* For more infos on the PCIe Page Request queue see chapter 3.3.
|
||||
*/
|
||||
struct riscv_iommu_pq_record {
|
||||
u64 hdr;
|
||||
u64 payload;
|
||||
};
|
||||
|
||||
/* Header fields */
|
||||
#define RISCV_IOMMU_PQ_HDR_PID GENMASK_ULL(31, 12)
|
||||
#define RISCV_IOMMU_PQ_HDR_PV BIT_ULL(32)
|
||||
#define RISCV_IOMMU_PQ_HDR_PRIV BIT_ULL(33)
|
||||
#define RISCV_IOMMU_PQ_HDR_EXEC BIT_ULL(34)
|
||||
#define RISCV_IOMMU_PQ_HDR_DID GENMASK_ULL(63, 40)
|
||||
|
||||
/* Payload fields */
|
||||
#define RISCV_IOMMU_PQ_PAYLOAD_R BIT_ULL(0)
|
||||
#define RISCV_IOMMU_PQ_PAYLOAD_W BIT_ULL(1)
|
||||
#define RISCV_IOMMU_PQ_PAYLOAD_L BIT_ULL(2)
|
||||
#define RISCV_IOMMU_PQ_PAYLOAD_RWL_MASK GENMASK_ULL(2, 0)
|
||||
#define RISCV_IOMMU_PQ_PAYLOAD_PRGI GENMASK_ULL(11, 3) /* Page Request Group Index */
|
||||
#define RISCV_IOMMU_PQ_PAYLOAD_ADDR GENMASK_ULL(63, 12)
|
||||
|
||||
/**
|
||||
* struct riscv_iommu_msipte - MSI Page Table Entry
|
||||
* @pte: MSI PTE
|
||||
* @mrif_info: Memory-resident interrupt file info
|
||||
*
|
||||
* The MSI Page Table is used for virtualizing MSIs, so that when
|
||||
* a device sends an MSI to a guest, the IOMMU can reroute it
|
||||
* by translating the MSI address, either to a guest interrupt file
|
||||
* or a memory resident interrupt file (MRIF). Note that this page table
|
||||
* is an array of MSI PTEs, not a multi-level pt, each entry
|
||||
* is a leaf entry. For more infos check out the AIA spec, chapter 9.5.
|
||||
*
|
||||
* Also in basic mode the mrif_info field is ignored by the IOMMU and can
|
||||
* be used by software, any other reserved fields on pte must be zeroed-out
|
||||
* by software.
|
||||
*/
|
||||
struct riscv_iommu_msipte {
|
||||
u64 pte;
|
||||
u64 mrif_info;
|
||||
};
|
||||
|
||||
/* Fields on pte */
|
||||
#define RISCV_IOMMU_MSIPTE_V BIT_ULL(0)
|
||||
#define RISCV_IOMMU_MSIPTE_M GENMASK_ULL(2, 1)
|
||||
#define RISCV_IOMMU_MSIPTE_MRIF_ADDR GENMASK_ULL(53, 7) /* When M == 1 (MRIF mode) */
|
||||
#define RISCV_IOMMU_MSIPTE_PPN RISCV_IOMMU_PPN_FIELD /* When M == 3 (basic mode) */
|
||||
#define RISCV_IOMMU_MSIPTE_C BIT_ULL(63)
|
||||
|
||||
/* Fields on mrif_info */
|
||||
#define RISCV_IOMMU_MSIPTE_MRIF_NID GENMASK_ULL(9, 0)
|
||||
#define RISCV_IOMMU_MSIPTE_MRIF_NPPN RISCV_IOMMU_PPN_FIELD
|
||||
#define RISCV_IOMMU_MSIPTE_MRIF_NID_MSB BIT_ULL(60)
|
||||
|
||||
/* Helper functions: command structure builders. */
|
||||
|
||||
static inline void riscv_iommu_cmd_inval_vma(struct riscv_iommu_command *cmd)
|
||||
{
|
||||
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOTINVAL_OPCODE) |
|
||||
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA);
|
||||
cmd->dword1 = 0;
|
||||
}
|
||||
|
||||
static inline void riscv_iommu_cmd_inval_set_addr(struct riscv_iommu_command *cmd,
|
||||
u64 addr)
|
||||
{
|
||||
cmd->dword1 = FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_ADDR, phys_to_pfn(addr));
|
||||
cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_AV;
|
||||
}
|
||||
|
||||
static inline void riscv_iommu_cmd_inval_set_pscid(struct riscv_iommu_command *cmd,
|
||||
int pscid)
|
||||
{
|
||||
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_PSCID, pscid) |
|
||||
RISCV_IOMMU_CMD_IOTINVAL_PSCV;
|
||||
}
|
||||
|
||||
static inline void riscv_iommu_cmd_inval_set_gscid(struct riscv_iommu_command *cmd,
|
||||
int gscid)
|
||||
{
|
||||
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_GSCID, gscid) |
|
||||
RISCV_IOMMU_CMD_IOTINVAL_GV;
|
||||
}
|
||||
|
||||
static inline void riscv_iommu_cmd_iofence(struct riscv_iommu_command *cmd)
|
||||
{
|
||||
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
|
||||
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
|
||||
RISCV_IOMMU_CMD_IOFENCE_PR | RISCV_IOMMU_CMD_IOFENCE_PW;
|
||||
cmd->dword1 = 0;
|
||||
}
|
||||
|
||||
static inline void riscv_iommu_cmd_iofence_set_av(struct riscv_iommu_command *cmd,
|
||||
u64 addr, u32 data)
|
||||
{
|
||||
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
|
||||
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
|
||||
FIELD_PREP(RISCV_IOMMU_CMD_IOFENCE_DATA, data) |
|
||||
RISCV_IOMMU_CMD_IOFENCE_AV;
|
||||
cmd->dword1 = addr >> 2;
|
||||
}
|
||||
|
||||
static inline void riscv_iommu_cmd_iodir_inval_ddt(struct riscv_iommu_command *cmd)
|
||||
{
|
||||
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
|
||||
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT);
|
||||
cmd->dword1 = 0;
|
||||
}
|
||||
|
||||
static inline void riscv_iommu_cmd_iodir_inval_pdt(struct riscv_iommu_command *cmd)
|
||||
{
|
||||
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
|
||||
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT);
|
||||
cmd->dword1 = 0;
|
||||
}
|
||||
|
||||
static inline void riscv_iommu_cmd_iodir_set_did(struct riscv_iommu_command *cmd,
|
||||
unsigned int devid)
|
||||
{
|
||||
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_DID, devid) |
|
||||
RISCV_IOMMU_CMD_IODIR_DV;
|
||||
}
|
||||
|
||||
static inline void riscv_iommu_cmd_iodir_set_pid(struct riscv_iommu_command *cmd,
|
||||
unsigned int pasid)
|
||||
{
|
||||
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_PID, pasid);
|
||||
}
|
||||
|
||||
#endif /* _RISCV_IOMMU_BITS_H_ */
|
120
drivers/iommu/riscv/iommu-pci.c
Normal file
120
drivers/iommu/riscv/iommu-pci.c
Normal file
@ -0,0 +1,120 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
|
||||
/*
|
||||
* Copyright © 2022-2024 Rivos Inc.
|
||||
* Copyright © 2023 FORTH-ICS/CARV
|
||||
*
|
||||
* RISCV IOMMU as a PCIe device
|
||||
*
|
||||
* Authors
|
||||
* Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||
* Nick Kossifidis <mick@ics.forth.gr>
|
||||
*/
|
||||
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/pci.h>
|
||||
|
||||
#include "iommu-bits.h"
|
||||
#include "iommu.h"
|
||||
|
||||
/* QEMU RISC-V IOMMU implementation */
|
||||
#define PCI_DEVICE_ID_REDHAT_RISCV_IOMMU 0x0014
|
||||
|
||||
/* Rivos Inc. assigned PCI Vendor and Device IDs */
|
||||
#ifndef PCI_VENDOR_ID_RIVOS
|
||||
#define PCI_VENDOR_ID_RIVOS 0x1efd
|
||||
#endif
|
||||
|
||||
#define PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA 0x0008
|
||||
|
||||
static int riscv_iommu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
||||
{
|
||||
struct device *dev = &pdev->dev;
|
||||
struct riscv_iommu_device *iommu;
|
||||
int rc, vec;
|
||||
|
||||
rc = pcim_enable_device(pdev);
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM))
|
||||
return -ENODEV;
|
||||
|
||||
if (pci_resource_len(pdev, 0) < RISCV_IOMMU_REG_SIZE)
|
||||
return -ENODEV;
|
||||
|
||||
rc = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
|
||||
if (rc)
|
||||
return dev_err_probe(dev, rc, "pcim_iomap_regions failed\n");
|
||||
|
||||
iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
|
||||
if (!iommu)
|
||||
return -ENOMEM;
|
||||
|
||||
iommu->dev = dev;
|
||||
iommu->reg = pcim_iomap_table(pdev)[0];
|
||||
|
||||
pci_set_master(pdev);
|
||||
dev_set_drvdata(dev, iommu);
|
||||
|
||||
/* Check device reported capabilities / features. */
|
||||
iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
|
||||
iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
|
||||
|
||||
/* The PCI driver only uses MSIs, make sure the IOMMU supports this */
|
||||
switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
|
||||
case RISCV_IOMMU_CAPABILITIES_IGS_MSI:
|
||||
case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
|
||||
break;
|
||||
default:
|
||||
return dev_err_probe(dev, -ENODEV,
|
||||
"unable to use message-signaled interrupts\n");
|
||||
}
|
||||
|
||||
/* Allocate and assign IRQ vectors for the various events */
|
||||
rc = pci_alloc_irq_vectors(pdev, 1, RISCV_IOMMU_INTR_COUNT,
|
||||
PCI_IRQ_MSIX | PCI_IRQ_MSI);
|
||||
if (rc <= 0)
|
||||
return dev_err_probe(dev, -ENODEV,
|
||||
"unable to allocate irq vectors\n");
|
||||
|
||||
iommu->irqs_count = rc;
|
||||
for (vec = 0; vec < iommu->irqs_count; vec++)
|
||||
iommu->irqs[vec] = msi_get_virq(dev, vec);
|
||||
|
||||
/* Enable message-signaled interrupts, fctl.WSI */
|
||||
if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {
|
||||
iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;
|
||||
riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
|
||||
}
|
||||
|
||||
return riscv_iommu_init(iommu);
|
||||
}
|
||||
|
||||
static void riscv_iommu_pci_remove(struct pci_dev *pdev)
|
||||
{
|
||||
struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);
|
||||
|
||||
riscv_iommu_remove(iommu);
|
||||
}
|
||||
|
||||
static const struct pci_device_id riscv_iommu_pci_tbl[] = {
|
||||
{PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_RISCV_IOMMU), 0},
|
||||
{PCI_VDEVICE(RIVOS, PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA), 0},
|
||||
{0,}
|
||||
};
|
||||
|
||||
static struct pci_driver riscv_iommu_pci_driver = {
|
||||
.name = KBUILD_MODNAME,
|
||||
.id_table = riscv_iommu_pci_tbl,
|
||||
.probe = riscv_iommu_pci_probe,
|
||||
.remove = riscv_iommu_pci_remove,
|
||||
.driver = {
|
||||
.suppress_bind_attrs = true,
|
||||
},
|
||||
};
|
||||
|
||||
builtin_pci_driver(riscv_iommu_pci_driver);
|
92
drivers/iommu/riscv/iommu-platform.c
Normal file
92
drivers/iommu/riscv/iommu-platform.c
Normal file
@ -0,0 +1,92 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* RISC-V IOMMU as a platform device
|
||||
*
|
||||
* Copyright © 2023 FORTH-ICS/CARV
|
||||
* Copyright © 2023-2024 Rivos Inc.
|
||||
*
|
||||
* Authors
|
||||
* Nick Kossifidis <mick@ics.forth.gr>
|
||||
* Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||
*/
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/of_platform.h>
|
||||
#include <linux/platform_device.h>
|
||||
|
||||
#include "iommu-bits.h"
|
||||
#include "iommu.h"
|
||||
|
||||
static int riscv_iommu_platform_probe(struct platform_device *pdev)
|
||||
{
|
||||
struct device *dev = &pdev->dev;
|
||||
struct riscv_iommu_device *iommu = NULL;
|
||||
struct resource *res = NULL;
|
||||
int vec;
|
||||
|
||||
iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
|
||||
if (!iommu)
|
||||
return -ENOMEM;
|
||||
|
||||
iommu->dev = dev;
|
||||
iommu->reg = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
|
||||
if (IS_ERR(iommu->reg))
|
||||
return dev_err_probe(dev, PTR_ERR(iommu->reg),
|
||||
"could not map register region\n");
|
||||
|
||||
dev_set_drvdata(dev, iommu);
|
||||
|
||||
/* Check device reported capabilities / features. */
|
||||
iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
|
||||
iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
|
||||
|
||||
/* For now we only support WSI */
|
||||
switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
|
||||
case RISCV_IOMMU_CAPABILITIES_IGS_WSI:
|
||||
case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
|
||||
break;
|
||||
default:
|
||||
return dev_err_probe(dev, -ENODEV,
|
||||
"unable to use wire-signaled interrupts\n");
|
||||
}
|
||||
|
||||
iommu->irqs_count = platform_irq_count(pdev);
|
||||
if (iommu->irqs_count <= 0)
|
||||
return dev_err_probe(dev, -ENODEV,
|
||||
"no IRQ resources provided\n");
|
||||
if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT)
|
||||
iommu->irqs_count = RISCV_IOMMU_INTR_COUNT;
|
||||
|
||||
for (vec = 0; vec < iommu->irqs_count; vec++)
|
||||
iommu->irqs[vec] = platform_get_irq(pdev, vec);
|
||||
|
||||
/* Enable wire-signaled interrupts, fctl.WSI */
|
||||
if (!(iommu->fctl & RISCV_IOMMU_FCTL_WSI)) {
|
||||
iommu->fctl |= RISCV_IOMMU_FCTL_WSI;
|
||||
riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
|
||||
}
|
||||
|
||||
return riscv_iommu_init(iommu);
|
||||
};
|
||||
|
||||
static void riscv_iommu_platform_remove(struct platform_device *pdev)
|
||||
{
|
||||
riscv_iommu_remove(dev_get_drvdata(&pdev->dev));
|
||||
};
|
||||
|
||||
static const struct of_device_id riscv_iommu_of_match[] = {
|
||||
{.compatible = "riscv,iommu",},
|
||||
{},
|
||||
};
|
||||
|
||||
static struct platform_driver riscv_iommu_platform_driver = {
|
||||
.probe = riscv_iommu_platform_probe,
|
||||
.remove_new = riscv_iommu_platform_remove,
|
||||
.driver = {
|
||||
.name = "riscv,iommu",
|
||||
.of_match_table = riscv_iommu_of_match,
|
||||
.suppress_bind_attrs = true,
|
||||
},
|
||||
};
|
||||
|
||||
builtin_platform_driver(riscv_iommu_platform_driver);
|
1661
drivers/iommu/riscv/iommu.c
Normal file
1661
drivers/iommu/riscv/iommu.c
Normal file
File diff suppressed because it is too large
Load Diff
88
drivers/iommu/riscv/iommu.h
Normal file
88
drivers/iommu/riscv/iommu.h
Normal file
@ -0,0 +1,88 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright © 2022-2024 Rivos Inc.
|
||||
* Copyright © 2023 FORTH-ICS/CARV
|
||||
*
|
||||
* Authors
|
||||
* Tomasz Jeznach <tjeznach@rivosinc.com>
|
||||
* Nick Kossifidis <mick@ics.forth.gr>
|
||||
*/
|
||||
|
||||
#ifndef _RISCV_IOMMU_H_
|
||||
#define _RISCV_IOMMU_H_
|
||||
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/iopoll.h>
|
||||
|
||||
#include "iommu-bits.h"
|
||||
|
||||
struct riscv_iommu_device;
|
||||
|
||||
struct riscv_iommu_queue {
|
||||
atomic_t prod; /* unbounded producer allocation index */
|
||||
atomic_t head; /* unbounded shadow ring buffer consumer index */
|
||||
atomic_t tail; /* unbounded shadow ring buffer producer index */
|
||||
unsigned int mask; /* index mask, queue length - 1 */
|
||||
unsigned int irq; /* allocated interrupt number */
|
||||
struct riscv_iommu_device *iommu; /* iommu device handling the queue when active */
|
||||
void *base; /* ring buffer kernel pointer */
|
||||
dma_addr_t phys; /* ring buffer physical address */
|
||||
u16 qbr; /* base register offset, head and tail reference */
|
||||
u16 qcr; /* control and status register offset */
|
||||
u8 qid; /* queue identifier, same as RISCV_IOMMU_INTR_XX */
|
||||
};
|
||||
|
||||
struct riscv_iommu_device {
|
||||
/* iommu core interface */
|
||||
struct iommu_device iommu;
|
||||
|
||||
/* iommu hardware */
|
||||
struct device *dev;
|
||||
|
||||
/* hardware control register space */
|
||||
void __iomem *reg;
|
||||
|
||||
/* supported and enabled hardware capabilities */
|
||||
u64 caps;
|
||||
u32 fctl;
|
||||
|
||||
/* available interrupt numbers, MSI or WSI */
|
||||
unsigned int irqs[RISCV_IOMMU_INTR_COUNT];
|
||||
unsigned int irqs_count;
|
||||
unsigned int icvec;
|
||||
|
||||
/* hardware queues */
|
||||
struct riscv_iommu_queue cmdq;
|
||||
struct riscv_iommu_queue fltq;
|
||||
|
||||
/* device directory */
|
||||
unsigned int ddt_mode;
|
||||
dma_addr_t ddt_phys;
|
||||
u64 *ddt_root;
|
||||
};
|
||||
|
||||
int riscv_iommu_init(struct riscv_iommu_device *iommu);
|
||||
void riscv_iommu_remove(struct riscv_iommu_device *iommu);
|
||||
|
||||
#define riscv_iommu_readl(iommu, addr) \
|
||||
readl_relaxed((iommu)->reg + (addr))
|
||||
|
||||
#define riscv_iommu_readq(iommu, addr) \
|
||||
readq_relaxed((iommu)->reg + (addr))
|
||||
|
||||
#define riscv_iommu_writel(iommu, addr, val) \
|
||||
writel_relaxed((val), (iommu)->reg + (addr))
|
||||
|
||||
#define riscv_iommu_writeq(iommu, addr, val) \
|
||||
writeq_relaxed((val), (iommu)->reg + (addr))
|
||||
|
||||
#define riscv_iommu_readq_timeout(iommu, addr, val, cond, delay_us, timeout_us) \
|
||||
readx_poll_timeout(readq_relaxed, (iommu)->reg + (addr), val, cond, \
|
||||
delay_us, timeout_us)
|
||||
|
||||
#define riscv_iommu_readl_timeout(iommu, addr, val, cond, delay_us, timeout_us) \
|
||||
readx_poll_timeout(readl_relaxed, (iommu)->reg + (addr), val, cond, \
|
||||
delay_us, timeout_us)
|
||||
|
||||
#endif
|
@ -33,6 +33,8 @@ struct s390_domain {
|
||||
struct rcu_head rcu;
|
||||
};
|
||||
|
||||
static struct iommu_domain blocking_domain;
|
||||
|
||||
static inline unsigned int calc_rtx(dma_addr_t ptr)
|
||||
{
|
||||
return ((unsigned long)ptr >> ZPCI_RT_SHIFT) & ZPCI_INDEX_MASK;
|
||||
@ -369,20 +371,36 @@ static void s390_domain_free(struct iommu_domain *domain)
|
||||
call_rcu(&s390_domain->rcu, s390_iommu_rcu_free_domain);
|
||||
}
|
||||
|
||||
static void s390_iommu_detach_device(struct iommu_domain *domain,
|
||||
struct device *dev)
|
||||
static void zdev_s390_domain_update(struct zpci_dev *zdev,
|
||||
struct iommu_domain *domain)
|
||||
{
|
||||
struct s390_domain *s390_domain = to_s390_domain(domain);
|
||||
struct zpci_dev *zdev = to_zpci_dev(dev);
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&zdev->dom_lock, flags);
|
||||
zdev->s390_domain = domain;
|
||||
spin_unlock_irqrestore(&zdev->dom_lock, flags);
|
||||
}
|
||||
|
||||
static int blocking_domain_attach_device(struct iommu_domain *domain,
|
||||
struct device *dev)
|
||||
{
|
||||
struct zpci_dev *zdev = to_zpci_dev(dev);
|
||||
struct s390_domain *s390_domain;
|
||||
unsigned long flags;
|
||||
|
||||
if (zdev->s390_domain->type == IOMMU_DOMAIN_BLOCKED)
|
||||
return 0;
|
||||
|
||||
s390_domain = to_s390_domain(zdev->s390_domain);
|
||||
spin_lock_irqsave(&s390_domain->list_lock, flags);
|
||||
list_del_rcu(&zdev->iommu_list);
|
||||
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
|
||||
|
||||
zpci_unregister_ioat(zdev, 0);
|
||||
zdev->s390_domain = NULL;
|
||||
zdev->dma_table = NULL;
|
||||
zdev_s390_domain_update(zdev, domain);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int s390_iommu_attach_device(struct iommu_domain *domain,
|
||||
@ -401,20 +419,15 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
|
||||
domain->geometry.aperture_end < zdev->start_dma))
|
||||
return -EINVAL;
|
||||
|
||||
if (zdev->s390_domain)
|
||||
s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
|
||||
blocking_domain_attach_device(&blocking_domain, dev);
|
||||
|
||||
/* If we fail now DMA remains blocked via blocking domain */
|
||||
cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
|
||||
virt_to_phys(s390_domain->dma_table), &status);
|
||||
/*
|
||||
* If the device is undergoing error recovery the reset code
|
||||
* will re-establish the new domain.
|
||||
*/
|
||||
if (cc && status != ZPCI_PCI_ST_FUNC_NOT_AVAIL)
|
||||
return -EIO;
|
||||
|
||||
zdev->dma_table = s390_domain->dma_table;
|
||||
zdev->s390_domain = s390_domain;
|
||||
zdev_s390_domain_update(zdev, domain);
|
||||
|
||||
spin_lock_irqsave(&s390_domain->list_lock, flags);
|
||||
list_add_rcu(&zdev->iommu_list, &s390_domain->devices);
|
||||
@ -466,21 +479,13 @@ static struct iommu_device *s390_iommu_probe_device(struct device *dev)
|
||||
if (zdev->tlb_refresh)
|
||||
dev->iommu->shadow_on_flush = 1;
|
||||
|
||||
/* Start with DMA blocked */
|
||||
spin_lock_init(&zdev->dom_lock);
|
||||
zdev_s390_domain_update(zdev, &blocking_domain);
|
||||
|
||||
return &zdev->iommu_dev;
|
||||
}
|
||||
|
||||
static void s390_iommu_release_device(struct device *dev)
|
||||
{
|
||||
struct zpci_dev *zdev = to_zpci_dev(dev);
|
||||
|
||||
/*
|
||||
* release_device is expected to detach any domain currently attached
|
||||
* to the device, but keep it attached to other devices in the group.
|
||||
*/
|
||||
if (zdev)
|
||||
s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
|
||||
}
|
||||
|
||||
static int zpci_refresh_all(struct zpci_dev *zdev)
|
||||
{
|
||||
return zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma,
|
||||
@ -697,9 +702,15 @@ static size_t s390_iommu_unmap_pages(struct iommu_domain *domain,
|
||||
|
||||
struct zpci_iommu_ctrs *zpci_get_iommu_ctrs(struct zpci_dev *zdev)
|
||||
{
|
||||
if (!zdev || !zdev->s390_domain)
|
||||
struct s390_domain *s390_domain;
|
||||
|
||||
lockdep_assert_held(&zdev->dom_lock);
|
||||
|
||||
if (zdev->s390_domain->type == IOMMU_DOMAIN_BLOCKED)
|
||||
return NULL;
|
||||
return &zdev->s390_domain->ctrs;
|
||||
|
||||
s390_domain = to_s390_domain(zdev->s390_domain);
|
||||
return &s390_domain->ctrs;
|
||||
}
|
||||
|
||||
int zpci_init_iommu(struct zpci_dev *zdev)
|
||||
@ -776,11 +787,19 @@ static int __init s390_iommu_init(void)
|
||||
}
|
||||
subsys_initcall(s390_iommu_init);
|
||||
|
||||
static struct iommu_domain blocking_domain = {
|
||||
.type = IOMMU_DOMAIN_BLOCKED,
|
||||
.ops = &(const struct iommu_domain_ops) {
|
||||
.attach_dev = blocking_domain_attach_device,
|
||||
}
|
||||
};
|
||||
|
||||
static const struct iommu_ops s390_iommu_ops = {
|
||||
.blocked_domain = &blocking_domain,
|
||||
.release_domain = &blocking_domain,
|
||||
.capable = s390_iommu_capable,
|
||||
.domain_alloc_paging = s390_domain_alloc_paging,
|
||||
.probe_device = s390_iommu_probe_device,
|
||||
.release_device = s390_iommu_release_device,
|
||||
.device_group = generic_device_group,
|
||||
.pgsize_bitmap = SZ_4K,
|
||||
.get_resv_regions = s390_iommu_get_resv_regions,
|
||||
|
@ -78,9 +78,10 @@ int tegra_vde_iommu_init(struct tegra_vde *vde)
|
||||
arm_iommu_release_mapping(mapping);
|
||||
}
|
||||
#endif
|
||||
vde->domain = iommu_domain_alloc(&platform_bus_type);
|
||||
if (!vde->domain) {
|
||||
err = -ENOMEM;
|
||||
vde->domain = iommu_paging_domain_alloc(dev);
|
||||
if (IS_ERR(vde->domain)) {
|
||||
err = PTR_ERR(vde->domain);
|
||||
vde->domain = NULL;
|
||||
goto put_group;
|
||||
}
|
||||
|
||||
|
@ -109,10 +109,10 @@ static int rproc_enable_iommu(struct rproc *rproc)
|
||||
return 0;
|
||||
}
|
||||
|
||||
domain = iommu_domain_alloc(dev->bus);
|
||||
if (!domain) {
|
||||
domain = iommu_paging_domain_alloc(dev);
|
||||
if (IS_ERR(domain)) {
|
||||
dev_err(dev, "can't alloc iommu domain\n");
|
||||
return -ENOMEM;
|
||||
return PTR_ERR(domain);
|
||||
}
|
||||
|
||||
iommu_set_fault_handler(domain, rproc_iommu_fault, rproc);
|
||||
|
@ -511,8 +511,6 @@ static inline int __iommu_copy_struct_from_user_array(
|
||||
* the caller iommu_domain_alloc() returns.
|
||||
* @domain_alloc_user: Allocate an iommu domain corresponding to the input
|
||||
* parameters as defined in include/uapi/linux/iommufd.h.
|
||||
* Unlike @domain_alloc, it is called only by IOMMUFD and
|
||||
* must fully initialize the new domain before return.
|
||||
* Upon success, if the @user_data is valid and the @parent
|
||||
* points to a kernel-managed domain, the new domain must be
|
||||
* IOMMU_DOMAIN_NESTED type; otherwise, the @parent must be
|
||||
@ -784,12 +782,14 @@ static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
|
||||
};
|
||||
}
|
||||
|
||||
extern int bus_iommu_probe(const struct bus_type *bus);
|
||||
extern bool iommu_present(const struct bus_type *bus);
|
||||
extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap);
|
||||
extern bool iommu_group_has_isolated_msi(struct iommu_group *group);
|
||||
extern struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus);
|
||||
struct iommu_domain *iommu_paging_domain_alloc(struct device *dev);
|
||||
struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev, unsigned int flags);
|
||||
static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
|
||||
{
|
||||
return iommu_paging_domain_alloc_flags(dev, 0);
|
||||
}
|
||||
extern void iommu_domain_free(struct iommu_domain *domain);
|
||||
extern int iommu_attach_device(struct iommu_domain *domain,
|
||||
struct device *dev);
|
||||
@ -1081,19 +1081,15 @@ struct iommu_iotlb_gather {};
|
||||
struct iommu_dirty_bitmap {};
|
||||
struct iommu_dirty_ops {};
|
||||
|
||||
static inline bool iommu_present(const struct bus_type *bus)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
|
||||
static inline struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev,
|
||||
unsigned int flags)
|
||||
{
|
||||
return NULL;
|
||||
return ERR_PTR(-ENODEV);
|
||||
}
|
||||
|
||||
static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
|
||||
|
@ -359,11 +359,19 @@ struct iommu_vfio_ioas {
|
||||
* enforced on device attachment
|
||||
* @IOMMU_HWPT_FAULT_ID_VALID: The fault_id field of hwpt allocation data is
|
||||
* valid.
|
||||
* @IOMMU_HWPT_ALLOC_PASID: Requests a domain that can be used with PASID. The
|
||||
* domain can be attached to any PASID on the device.
|
||||
* Any domain attached to the non-PASID part of the
|
||||
* device must also be flaged, otherwise attaching a
|
||||
* PASID will blocked.
|
||||
* If IOMMU does not support PASID it will return
|
||||
* error (-EOPNOTSUPP).
|
||||
*/
|
||||
enum iommufd_hwpt_alloc_flags {
|
||||
IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0,
|
||||
IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1,
|
||||
IOMMU_HWPT_FAULT_ID_VALID = 1 << 2,
|
||||
IOMMU_HWPT_ALLOC_PASID = 1 << 3,
|
||||
};
|
||||
|
||||
/**
|
||||
|
Loading…
Reference in New Issue
Block a user