mirror of
https://github.com/torvalds/linux.git
synced 2024-11-16 17:12:06 +00:00
IOMMU Updates for Linux v5.17
Including:
- Identity domain support for virtio-iommu
- Move flush queue code into iommu-dma
- Some fixes for AMD IOMMU suspend/resume support when x2apic
is used
- Arm SMMU Updates from Will Deacon:
- Revert evtq and priq back to their former sizes
- Return early on short-descriptor page-table allocation failure
- Fix page fault reporting for Adreno GPU on SMMUv2
- Make SMMUv3 MMU notifier ops 'const'
- Numerous new compatible strings for Qualcomm SMMUv2 implementations
- Various smaller fixes and cleanups
-----BEGIN PGP SIGNATURE-----
iQIzBAABCAAdFiEEr9jSbILcajRFYWYyK/BELZcBGuMFAmHfBvsACgkQK/BELZcB
GuOURRAAqHBTUUgT/f2dJbVsWMOxSx0dhDnKuLRhAREMbmcn88tR8dxPRPYB1/6q
yyFmkh13UzEr1gosEd34P5G8GS6fMD/G4yz8p9tK6Mqu7UeU+DoiDIi3s194WhYa
iNPBlGgQ3XznTrbBnFV+n/LjvkxSguNfjj809Jiw7Ew3i50K4GFnzRA+IZVAT4+F
zdNmwY12Y0v4SCfHKiVR1ZB9kcn2/Dx78dlBQ6ALFuejeZGa2OVr94byKnfz+AJh
OssrgRcgUKclWbMk4Tljf4/FIdwkLMKD6gieBFb3sNKTUpzkG8elYmETO5d+hM+l
27CKOCz1OOqVRzKe3r5n3c0wf9UAmi0Q91zW+UVZp2i0GjxpfIeIS9/NwRy+IXHS
U9zybU47q10WF6cVO0n6wWHPRbjPii2OZpjqhSTq57qsnniCPLwkrry9H2fP71zz
NDAZv5qvHCvRF7QoZfkBvCCJ12ZhNnhZqTfZR2wGGITMIk6dokG4NCsU93rSVKvZ
4xQDPm45rECmunibdc9c1vrifKC7BIWCSU5DH3AEDBU/i9QfYpVPXfJlGdz3enIV
/FA+kcvYrh21sokly/TqiZXGSaOFBqFEN13KJReXOgbENNq6kT/4lSNjK5Q1WSWp
qDq12EQyv0RtTEcKVDpRVunI+/G5MquO8gbIrVmsRV0SU1Z0yoM=
=Q8LV
-----END PGP SIGNATURE-----
Merge tag 'iommu-updates-v5.17' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu
Pull iommu updates from Joerg Roedel:
- Identity domain support for virtio-iommu
- Move flush queue code into iommu-dma
- Some fixes for AMD IOMMU suspend/resume support when x2apic is used
- Arm SMMU Updates from Will Deacon:
- Revert evtq and priq back to their former sizes
- Return early on short-descriptor page-table allocation failure
- Fix page fault reporting for Adreno GPU on SMMUv2
- Make SMMUv3 MMU notifier ops 'const'
- Numerous new compatible strings for Qualcomm SMMUv2 implementations
- Various smaller fixes and cleanups
* tag 'iommu-updates-v5.17' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (38 commits)
iommu/iova: Temporarily include dma-mapping.h from iova.h
iommu: Move flush queue data into iommu_dma_cookie
iommu/iova: Move flush queue code to iommu-dma
iommu/iova: Consolidate flush queue code
iommu/vt-d: Use put_pages_list
iommu/amd: Use put_pages_list
iommu/amd: Simplify pagetable freeing
iommu/iova: Squash flush_cb abstraction
iommu/iova: Squash entry_dtor abstraction
iommu/iova: Fix race between FQ timeout and teardown
iommu/amd: Fix typo in *glues … together* in comment
iommu/vt-d: Remove unused dma_to_mm_pfn function
iommu/vt-d: Drop duplicate check in dma_pte_free_pagetable()
iommu/vt-d: Use bitmap_zalloc() when applicable
iommu/amd: Remove useless irq affinity notifier
iommu/amd: X2apic mode: mask/unmask interrupts on suspend/resume
iommu/amd: X2apic mode: setup the INTX registers on mask/unmask
iommu/amd: X2apic mode: re-enable after resume
iommu/amd: Restore GA log/tail pointer on host resume
iommu/iova: Move fast alloc size roundup into alloc_iova_fast()
...
This commit is contained in:
Linus Torvalds
commit
13eaa5bda0
@ -38,10 +38,12 @@ properties:
|
||||
- qcom,sc7280-smmu-500
|
||||
- qcom,sc8180x-smmu-500
|
||||
- qcom,sdm845-smmu-500
|
||||
- qcom,sdx55-smmu-500
|
||||
- qcom,sm6350-smmu-500
|
||||
- qcom,sm8150-smmu-500
|
||||
- qcom,sm8250-smmu-500
|
||||
- qcom,sm8350-smmu-500
|
||||
- qcom,sm8450-smmu-500
|
||||
- const: arm,mmu-500
|
||||
- description: Qcom Adreno GPUs implementing "arm,smmu-v2"
|
||||
items:
|
||||
|
||||
@ -645,8 +645,6 @@ struct amd_iommu {
|
||||
/* DebugFS Info */
|
||||
struct dentry *debugfs;
|
||||
#endif
|
||||
/* IRQ notifier for IntCapXT interrupt */
|
||||
struct irq_affinity_notify intcapxt_notify;
|
||||
};
|
||||
|
||||
static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)
|
||||
|
||||
@ -806,16 +806,27 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
|
||||
{
|
||||
#ifdef CONFIG_IRQ_REMAP
|
||||
u32 status, i;
|
||||
u64 entry;
|
||||
|
||||
if (!iommu->ga_log)
|
||||
return -EINVAL;
|
||||
|
||||
status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
|
||||
|
||||
/* Check if already running */
|
||||
if (status & (MMIO_STATUS_GALOG_RUN_MASK))
|
||||
status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
|
||||
if (WARN_ON(status & (MMIO_STATUS_GALOG_RUN_MASK)))
|
||||
return 0;
|
||||
|
||||
entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
|
||||
memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
|
||||
&entry, sizeof(entry));
|
||||
entry = (iommu_virt_to_phys(iommu->ga_log_tail) &
|
||||
(BIT_ULL(52)-1)) & ~7ULL;
|
||||
memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
|
||||
&entry, sizeof(entry));
|
||||
writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
|
||||
writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
|
||||
|
||||
|
||||
iommu_feature_enable(iommu, CONTROL_GAINT_EN);
|
||||
iommu_feature_enable(iommu, CONTROL_GALOG_EN);
|
||||
|
||||
@ -825,7 +836,7 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
|
||||
break;
|
||||
}
|
||||
|
||||
if (i >= LOOP_TIMEOUT)
|
||||
if (WARN_ON(i >= LOOP_TIMEOUT))
|
||||
return -EINVAL;
|
||||
#endif /* CONFIG_IRQ_REMAP */
|
||||
return 0;
|
||||
@ -834,8 +845,6 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
|
||||
static int iommu_init_ga_log(struct amd_iommu *iommu)
|
||||
{
|
||||
#ifdef CONFIG_IRQ_REMAP
|
||||
u64 entry;
|
||||
|
||||
if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
|
||||
return 0;
|
||||
|
||||
@ -849,16 +858,6 @@ static int iommu_init_ga_log(struct amd_iommu *iommu)
|
||||
if (!iommu->ga_log_tail)
|
||||
goto err_out;
|
||||
|
||||
entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
|
||||
memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
|
||||
&entry, sizeof(entry));
|
||||
entry = (iommu_virt_to_phys(iommu->ga_log_tail) &
|
||||
(BIT_ULL(52)-1)) & ~7ULL;
|
||||
memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
|
||||
&entry, sizeof(entry));
|
||||
writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
|
||||
writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
|
||||
|
||||
return 0;
|
||||
err_out:
|
||||
free_ga_log(iommu);
|
||||
@ -1523,7 +1522,7 @@ static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu)
|
||||
}
|
||||
|
||||
/*
|
||||
* This function clues the initialization function for one IOMMU
|
||||
* This function glues the initialization function for one IOMMU
|
||||
* together and also allocates the command buffer and programs the
|
||||
* hardware. It does NOT enable the IOMMU. This is done afterwards.
|
||||
*/
|
||||
@ -2016,48 +2015,18 @@ union intcapxt {
|
||||
};
|
||||
} __attribute__ ((packed));
|
||||
|
||||
/*
|
||||
* There isn't really any need to mask/unmask at the irqchip level because
|
||||
* the 64-bit INTCAPXT registers can be updated atomically without tearing
|
||||
* when the affinity is being updated.
|
||||
*/
|
||||
static void intcapxt_unmask_irq(struct irq_data *data)
|
||||
{
|
||||
}
|
||||
|
||||
static void intcapxt_mask_irq(struct irq_data *data)
|
||||
{
|
||||
}
|
||||
|
||||
static struct irq_chip intcapxt_controller;
|
||||
|
||||
static int intcapxt_irqdomain_activate(struct irq_domain *domain,
|
||||
struct irq_data *irqd, bool reserve)
|
||||
{
|
||||
struct amd_iommu *iommu = irqd->chip_data;
|
||||
struct irq_cfg *cfg = irqd_cfg(irqd);
|
||||
union intcapxt xt;
|
||||
|
||||
xt.capxt = 0ULL;
|
||||
xt.dest_mode_logical = apic->dest_mode_logical;
|
||||
xt.vector = cfg->vector;
|
||||
xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);
|
||||
xt.destid_24_31 = cfg->dest_apicid >> 24;
|
||||
|
||||
/**
|
||||
* Current IOMMU implemtation uses the same IRQ for all
|
||||
* 3 IOMMU interrupts.
|
||||
*/
|
||||
writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
|
||||
writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
|
||||
writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void intcapxt_irqdomain_deactivate(struct irq_domain *domain,
|
||||
struct irq_data *irqd)
|
||||
{
|
||||
intcapxt_mask_irq(irqd);
|
||||
}
|
||||
|
||||
|
||||
@ -2091,6 +2060,38 @@ static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq
|
||||
irq_domain_free_irqs_top(domain, virq, nr_irqs);
|
||||
}
|
||||
|
||||
|
||||
static void intcapxt_unmask_irq(struct irq_data *irqd)
|
||||
{
|
||||
struct amd_iommu *iommu = irqd->chip_data;
|
||||
struct irq_cfg *cfg = irqd_cfg(irqd);
|
||||
union intcapxt xt;
|
||||
|
||||
xt.capxt = 0ULL;
|
||||
xt.dest_mode_logical = apic->dest_mode_logical;
|
||||
xt.vector = cfg->vector;
|
||||
xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);
|
||||
xt.destid_24_31 = cfg->dest_apicid >> 24;
|
||||
|
||||
/**
|
||||
* Current IOMMU implementation uses the same IRQ for all
|
||||
* 3 IOMMU interrupts.
|
||||
*/
|
||||
writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
|
||||
writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
|
||||
writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
|
||||
}
|
||||
|
||||
static void intcapxt_mask_irq(struct irq_data *irqd)
|
||||
{
|
||||
struct amd_iommu *iommu = irqd->chip_data;
|
||||
|
||||
writeq(0, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
|
||||
writeq(0, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
|
||||
writeq(0, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
|
||||
}
|
||||
|
||||
|
||||
static int intcapxt_set_affinity(struct irq_data *irqd,
|
||||
const struct cpumask *mask, bool force)
|
||||
{
|
||||
@ -2100,8 +2101,12 @@ static int intcapxt_set_affinity(struct irq_data *irqd,
|
||||
ret = parent->chip->irq_set_affinity(parent, mask, force);
|
||||
if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
|
||||
return ret;
|
||||
return 0;
|
||||
}
|
||||
|
||||
return intcapxt_irqdomain_activate(irqd->domain, irqd, false);
|
||||
static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on)
|
||||
{
|
||||
return on ? -EOPNOTSUPP : 0;
|
||||
}
|
||||
|
||||
static struct irq_chip intcapxt_controller = {
|
||||
@ -2111,7 +2116,8 @@ static struct irq_chip intcapxt_controller = {
|
||||
.irq_ack = irq_chip_ack_parent,
|
||||
.irq_retrigger = irq_chip_retrigger_hierarchy,
|
||||
.irq_set_affinity = intcapxt_set_affinity,
|
||||
.flags = IRQCHIP_SKIP_SET_WAKE,
|
||||
.irq_set_wake = intcapxt_set_wake,
|
||||
.flags = IRQCHIP_MASK_ON_SUSPEND,
|
||||
};
|
||||
|
||||
static const struct irq_domain_ops intcapxt_domain_ops = {
|
||||
@ -2173,7 +2179,6 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu)
|
||||
return ret;
|
||||
}
|
||||
|
||||
iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -2196,6 +2201,10 @@ static int iommu_init_irq(struct amd_iommu *iommu)
|
||||
|
||||
iommu->int_enabled = true;
|
||||
enable_faults:
|
||||
|
||||
if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
|
||||
iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
|
||||
|
||||
iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
|
||||
|
||||
if (iommu->ppr_log != NULL)
|
||||
|
||||
@ -74,87 +74,61 @@ static u64 *first_pte_l7(u64 *pte, unsigned long *page_size,
|
||||
*
|
||||
****************************************************************************/
|
||||
|
||||
static void free_page_list(struct page *freelist)
|
||||
static void free_pt_page(u64 *pt, struct list_head *freelist)
|
||||
{
|
||||
while (freelist != NULL) {
|
||||
unsigned long p = (unsigned long)page_address(freelist);
|
||||
struct page *p = virt_to_page(pt);
|
||||
|
||||
freelist = freelist->freelist;
|
||||
free_page(p);
|
||||
list_add_tail(&p->lru, freelist);
|
||||
}
|
||||
|
||||
static void free_pt_lvl(u64 *pt, struct list_head *freelist, int lvl)
|
||||
{
|
||||
u64 *p;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 512; ++i) {
|
||||
/* PTE present? */
|
||||
if (!IOMMU_PTE_PRESENT(pt[i]))
|
||||
continue;
|
||||
|
||||
/* Large PTE? */
|
||||
if (PM_PTE_LEVEL(pt[i]) == 0 ||
|
||||
PM_PTE_LEVEL(pt[i]) == 7)
|
||||
continue;
|
||||
|
||||
/*
|
||||
* Free the next level. No need to look at l1 tables here since
|
||||
* they can only contain leaf PTEs; just free them directly.
|
||||
*/
|
||||
p = IOMMU_PTE_PAGE(pt[i]);
|
||||
if (lvl > 2)
|
||||
free_pt_lvl(p, freelist, lvl - 1);
|
||||
else
|
||||
free_pt_page(p, freelist);
|
||||
}
|
||||
|
||||
free_pt_page(pt, freelist);
|
||||
}
|
||||
|
||||
static struct page *free_pt_page(unsigned long pt, struct page *freelist)
|
||||
{
|
||||
struct page *p = virt_to_page((void *)pt);
|
||||
|
||||
p->freelist = freelist;
|
||||
|
||||
return p;
|
||||
}
|
||||
|
||||
#define DEFINE_FREE_PT_FN(LVL, FN) \
|
||||
static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist) \
|
||||
{ \
|
||||
unsigned long p; \
|
||||
u64 *pt; \
|
||||
int i; \
|
||||
\
|
||||
pt = (u64 *)__pt; \
|
||||
\
|
||||
for (i = 0; i < 512; ++i) { \
|
||||
/* PTE present? */ \
|
||||
if (!IOMMU_PTE_PRESENT(pt[i])) \
|
||||
continue; \
|
||||
\
|
||||
/* Large PTE? */ \
|
||||
if (PM_PTE_LEVEL(pt[i]) == 0 || \
|
||||
PM_PTE_LEVEL(pt[i]) == 7) \
|
||||
continue; \
|
||||
\
|
||||
p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \
|
||||
freelist = FN(p, freelist); \
|
||||
} \
|
||||
\
|
||||
return free_pt_page((unsigned long)pt, freelist); \
|
||||
}
|
||||
|
||||
DEFINE_FREE_PT_FN(l2, free_pt_page)
|
||||
DEFINE_FREE_PT_FN(l3, free_pt_l2)
|
||||
DEFINE_FREE_PT_FN(l4, free_pt_l3)
|
||||
DEFINE_FREE_PT_FN(l5, free_pt_l4)
|
||||
DEFINE_FREE_PT_FN(l6, free_pt_l5)
|
||||
|
||||
static struct page *free_sub_pt(unsigned long root, int mode,
|
||||
struct page *freelist)
|
||||
static void free_sub_pt(u64 *root, int mode, struct list_head *freelist)
|
||||
{
|
||||
switch (mode) {
|
||||
case PAGE_MODE_NONE:
|
||||
case PAGE_MODE_7_LEVEL:
|
||||
break;
|
||||
case PAGE_MODE_1_LEVEL:
|
||||
freelist = free_pt_page(root, freelist);
|
||||
free_pt_page(root, freelist);
|
||||
break;
|
||||
case PAGE_MODE_2_LEVEL:
|
||||
freelist = free_pt_l2(root, freelist);
|
||||
break;
|
||||
case PAGE_MODE_3_LEVEL:
|
||||
freelist = free_pt_l3(root, freelist);
|
||||
break;
|
||||
case PAGE_MODE_4_LEVEL:
|
||||
freelist = free_pt_l4(root, freelist);
|
||||
break;
|
||||
case PAGE_MODE_5_LEVEL:
|
||||
freelist = free_pt_l5(root, freelist);
|
||||
break;
|
||||
case PAGE_MODE_6_LEVEL:
|
||||
freelist = free_pt_l6(root, freelist);
|
||||
free_pt_lvl(root, freelist, mode);
|
||||
break;
|
||||
default:
|
||||
BUG();
|
||||
}
|
||||
|
||||
return freelist;
|
||||
}
|
||||
|
||||
void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
|
||||
@ -362,9 +336,9 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable,
|
||||
return pte;
|
||||
}
|
||||
|
||||
static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)
|
||||
static void free_clear_pte(u64 *pte, u64 pteval, struct list_head *freelist)
|
||||
{
|
||||
unsigned long pt;
|
||||
u64 *pt;
|
||||
int mode;
|
||||
|
||||
while (cmpxchg64(pte, pteval, 0) != pteval) {
|
||||
@ -373,12 +347,12 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)
|
||||
}
|
||||
|
||||
if (!IOMMU_PTE_PRESENT(pteval))
|
||||
return freelist;
|
||||
return;
|
||||
|
||||
pt = (unsigned long)IOMMU_PTE_PAGE(pteval);
|
||||
pt = IOMMU_PTE_PAGE(pteval);
|
||||
mode = IOMMU_PTE_MODE(pteval);
|
||||
|
||||
return free_sub_pt(pt, mode, freelist);
|
||||
free_sub_pt(pt, mode, freelist);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -392,7 +366,7 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
|
||||
{
|
||||
struct protection_domain *dom = io_pgtable_ops_to_domain(ops);
|
||||
struct page *freelist = NULL;
|
||||
LIST_HEAD(freelist);
|
||||
bool updated = false;
|
||||
u64 __pte, *pte;
|
||||
int ret, i, count;
|
||||
@ -412,9 +386,9 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
goto out;
|
||||
|
||||
for (i = 0; i < count; ++i)
|
||||
freelist = free_clear_pte(&pte[i], pte[i], freelist);
|
||||
free_clear_pte(&pte[i], pte[i], &freelist);
|
||||
|
||||
if (freelist != NULL)
|
||||
if (!list_empty(&freelist))
|
||||
updated = true;
|
||||
|
||||
if (count > 1) {
|
||||
@ -449,7 +423,7 @@ out:
|
||||
}
|
||||
|
||||
/* Everything flushed out, free pages now */
|
||||
free_page_list(freelist);
|
||||
put_pages_list(&freelist);
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -511,8 +485,7 @@ static void v1_free_pgtable(struct io_pgtable *iop)
|
||||
{
|
||||
struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop);
|
||||
struct protection_domain *dom;
|
||||
struct page *freelist = NULL;
|
||||
unsigned long root;
|
||||
LIST_HEAD(freelist);
|
||||
|
||||
if (pgtable->mode == PAGE_MODE_NONE)
|
||||
return;
|
||||
@ -529,10 +502,9 @@ static void v1_free_pgtable(struct io_pgtable *iop)
|
||||
BUG_ON(pgtable->mode < PAGE_MODE_NONE ||
|
||||
pgtable->mode > PAGE_MODE_6_LEVEL);
|
||||
|
||||
root = (unsigned long)pgtable->root;
|
||||
freelist = free_sub_pt(root, pgtable->mode, freelist);
|
||||
free_sub_pt(pgtable->root, pgtable->mode, &freelist);
|
||||
|
||||
free_page_list(freelist);
|
||||
put_pages_list(&freelist);
|
||||
}
|
||||
|
||||
static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
|
||||
|
||||
@ -220,7 +220,7 @@ static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn)
|
||||
kfree(mn_to_smmu(mn));
|
||||
}
|
||||
|
||||
static struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
|
||||
static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
|
||||
.invalidate_range = arm_smmu_mm_invalidate_range,
|
||||
.release = arm_smmu_mm_release,
|
||||
.free_notifier = arm_smmu_mmu_notifier_free,
|
||||
|
||||
@ -184,7 +184,6 @@
|
||||
#else
|
||||
#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
|
||||
#endif
|
||||
#define Q_MIN_SZ_SHIFT (PAGE_SHIFT)
|
||||
|
||||
/*
|
||||
* Stream table.
|
||||
@ -374,7 +373,7 @@
|
||||
/* Event queue */
|
||||
#define EVTQ_ENT_SZ_SHIFT 5
|
||||
#define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
|
||||
#define EVTQ_MAX_SZ_SHIFT (Q_MIN_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
|
||||
#define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
|
||||
|
||||
#define EVTQ_0_ID GENMASK_ULL(7, 0)
|
||||
|
||||
@ -400,7 +399,7 @@
|
||||
/* PRI queue */
|
||||
#define PRIQ_ENT_SZ_SHIFT 4
|
||||
#define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
|
||||
#define PRIQ_MAX_SZ_SHIFT (Q_MIN_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
|
||||
#define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
|
||||
|
||||
#define PRIQ_0_SID GENMASK_ULL(31, 0)
|
||||
#define PRIQ_0_SSID GENMASK_ULL(51, 32)
|
||||
|
||||
@ -51,7 +51,7 @@ static void qcom_adreno_smmu_get_fault_info(const void *cookie,
|
||||
info->fsynr1 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR1);
|
||||
info->far = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_FAR);
|
||||
info->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
|
||||
info->ttbr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0);
|
||||
info->ttbr0 = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0);
|
||||
info->contextidr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_CONTEXTIDR);
|
||||
}
|
||||
|
||||
@ -415,6 +415,7 @@ static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
|
||||
{ .compatible = "qcom,sm8150-smmu-500" },
|
||||
{ .compatible = "qcom,sm8250-smmu-500" },
|
||||
{ .compatible = "qcom,sm8350-smmu-500" },
|
||||
{ .compatible = "qcom,sm8450-smmu-500" },
|
||||
{ }
|
||||
};
|
||||
|
||||
|
||||
@ -9,9 +9,12 @@
|
||||
*/
|
||||
|
||||
#include <linux/acpi_iort.h>
|
||||
#include <linux/atomic.h>
|
||||
#include <linux/crash_dump.h>
|
||||
#include <linux/device.h>
|
||||
#include <linux/dma-map-ops.h>
|
||||
#include <linux/dma-direct.h>
|
||||
#include <linux/dma-iommu.h>
|
||||
#include <linux/dma-map-ops.h>
|
||||
#include <linux/gfp.h>
|
||||
#include <linux/huge_mm.h>
|
||||
#include <linux/iommu.h>
|
||||
@ -20,11 +23,10 @@
|
||||
#include <linux/mm.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/swiotlb.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/swiotlb.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/crash_dump.h>
|
||||
#include <linux/dma-direct.h>
|
||||
|
||||
struct iommu_dma_msi_page {
|
||||
struct list_head list;
|
||||
@ -41,7 +43,19 @@ struct iommu_dma_cookie {
|
||||
enum iommu_dma_cookie_type type;
|
||||
union {
|
||||
/* Full allocator for IOMMU_DMA_IOVA_COOKIE */
|
||||
struct iova_domain iovad;
|
||||
struct {
|
||||
struct iova_domain iovad;
|
||||
|
||||
struct iova_fq __percpu *fq; /* Flush queue */
|
||||
/* Number of TLB flushes that have been started */
|
||||
atomic64_t fq_flush_start_cnt;
|
||||
/* Number of TLB flushes that have been finished */
|
||||
atomic64_t fq_flush_finish_cnt;
|
||||
/* Timer to regularily empty the flush queues */
|
||||
struct timer_list fq_timer;
|
||||
/* 1 when timer is active, 0 when not */
|
||||
atomic_t fq_timer_on;
|
||||
};
|
||||
/* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
|
||||
dma_addr_t msi_iova;
|
||||
};
|
||||
@ -64,18 +78,205 @@ static int __init iommu_dma_forcedac_setup(char *str)
|
||||
}
|
||||
early_param("iommu.forcedac", iommu_dma_forcedac_setup);
|
||||
|
||||
static void iommu_dma_entry_dtor(unsigned long data)
|
||||
/* Number of entries per flush queue */
|
||||
#define IOVA_FQ_SIZE 256
|
||||
|
||||
/* Timeout (in ms) after which entries are flushed from the queue */
|
||||
#define IOVA_FQ_TIMEOUT 10
|
||||
|
||||
/* Flush queue entry for deferred flushing */
|
||||
struct iova_fq_entry {
|
||||
unsigned long iova_pfn;
|
||||
unsigned long pages;
|
||||
struct list_head freelist;
|
||||
u64 counter; /* Flush counter when this entry was added */
|
||||
};
|
||||
|
||||
/* Per-CPU flush queue structure */
|
||||
struct iova_fq {
|
||||
struct iova_fq_entry entries[IOVA_FQ_SIZE];
|
||||
unsigned int head, tail;
|
||||
spinlock_t lock;
|
||||
};
|
||||
|
||||
#define fq_ring_for_each(i, fq) \
|
||||
for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
|
||||
|
||||
static inline bool fq_full(struct iova_fq *fq)
|
||||
{
|
||||
struct page *freelist = (struct page *)data;
|
||||
assert_spin_locked(&fq->lock);
|
||||
return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
|
||||
}
|
||||
|
||||
while (freelist) {
|
||||
unsigned long p = (unsigned long)page_address(freelist);
|
||||
static inline unsigned int fq_ring_add(struct iova_fq *fq)
|
||||
{
|
||||
unsigned int idx = fq->tail;
|
||||
|
||||
freelist = freelist->freelist;
|
||||
free_page(p);
|
||||
assert_spin_locked(&fq->lock);
|
||||
|
||||
fq->tail = (idx + 1) % IOVA_FQ_SIZE;
|
||||
|
||||
return idx;
|
||||
}
|
||||
|
||||
static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq)
|
||||
{
|
||||
u64 counter = atomic64_read(&cookie->fq_flush_finish_cnt);
|
||||
unsigned int idx;
|
||||
|
||||
assert_spin_locked(&fq->lock);
|
||||
|
||||
fq_ring_for_each(idx, fq) {
|
||||
|
||||
if (fq->entries[idx].counter >= counter)
|
||||
break;
|
||||
|
||||
put_pages_list(&fq->entries[idx].freelist);
|
||||
free_iova_fast(&cookie->iovad,
|
||||
fq->entries[idx].iova_pfn,
|
||||
fq->entries[idx].pages);
|
||||
|
||||
fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void fq_flush_iotlb(struct iommu_dma_cookie *cookie)
|
||||
{
|
||||
atomic64_inc(&cookie->fq_flush_start_cnt);
|
||||
cookie->fq_domain->ops->flush_iotlb_all(cookie->fq_domain);
|
||||
atomic64_inc(&cookie->fq_flush_finish_cnt);
|
||||
}
|
||||
|
||||
static void fq_flush_timeout(struct timer_list *t)
|
||||
{
|
||||
struct iommu_dma_cookie *cookie = from_timer(cookie, t, fq_timer);
|
||||
int cpu;
|
||||
|
||||
atomic_set(&cookie->fq_timer_on, 0);
|
||||
fq_flush_iotlb(cookie);
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
unsigned long flags;
|
||||
struct iova_fq *fq;
|
||||
|
||||
fq = per_cpu_ptr(cookie->fq, cpu);
|
||||
spin_lock_irqsave(&fq->lock, flags);
|
||||
fq_ring_free(cookie, fq);
|
||||
spin_unlock_irqrestore(&fq->lock, flags);
|
||||
}
|
||||
}
|
||||
|
||||
static void queue_iova(struct iommu_dma_cookie *cookie,
|
||||
unsigned long pfn, unsigned long pages,
|
||||
struct list_head *freelist)
|
||||
{
|
||||
struct iova_fq *fq;
|
||||
unsigned long flags;
|
||||
unsigned int idx;
|
||||
|
||||
/*
|
||||
* Order against the IOMMU driver's pagetable update from unmapping
|
||||
* @pte, to guarantee that fq_flush_iotlb() observes that if called
|
||||
* from a different CPU before we release the lock below. Full barrier
|
||||
* so it also pairs with iommu_dma_init_fq() to avoid seeing partially
|
||||
* written fq state here.
|
||||
*/
|
||||
smp_mb();
|
||||
|
||||
fq = raw_cpu_ptr(cookie->fq);
|
||||
spin_lock_irqsave(&fq->lock, flags);
|
||||
|
||||
/*
|
||||
* First remove all entries from the flush queue that have already been
|
||||
* flushed out on another CPU. This makes the fq_full() check below less
|
||||
* likely to be true.
|
||||
*/
|
||||
fq_ring_free(cookie, fq);
|
||||
|
||||
if (fq_full(fq)) {
|
||||
fq_flush_iotlb(cookie);
|
||||
fq_ring_free(cookie, fq);
|
||||
}
|
||||
|
||||
idx = fq_ring_add(fq);
|
||||
|
||||
fq->entries[idx].iova_pfn = pfn;
|
||||
fq->entries[idx].pages = pages;
|
||||
fq->entries[idx].counter = atomic64_read(&cookie->fq_flush_start_cnt);
|
||||
list_splice(freelist, &fq->entries[idx].freelist);
|
||||
|
||||
spin_unlock_irqrestore(&fq->lock, flags);
|
||||
|
||||
/* Avoid false sharing as much as possible. */
|
||||
if (!atomic_read(&cookie->fq_timer_on) &&
|
||||
!atomic_xchg(&cookie->fq_timer_on, 1))
|
||||
mod_timer(&cookie->fq_timer,
|
||||
jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
|
||||
}
|
||||
|
||||
static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie)
|
||||
{
|
||||
int cpu, idx;
|
||||
|
||||
if (!cookie->fq)
|
||||
return;
|
||||
|
||||
del_timer_sync(&cookie->fq_timer);
|
||||
/* The IOVAs will be torn down separately, so just free our queued pages */
|
||||
for_each_possible_cpu(cpu) {
|
||||
struct iova_fq *fq = per_cpu_ptr(cookie->fq, cpu);
|
||||
|
||||
fq_ring_for_each(idx, fq)
|
||||
put_pages_list(&fq->entries[idx].freelist);
|
||||
}
|
||||
|
||||
free_percpu(cookie->fq);
|
||||
}
|
||||
|
||||
/* sysfs updates are serialised by the mutex of the group owning @domain */
|
||||
int iommu_dma_init_fq(struct iommu_domain *domain)
|
||||
{
|
||||
struct iommu_dma_cookie *cookie = domain->iova_cookie;
|
||||
struct iova_fq __percpu *queue;
|
||||
int i, cpu;
|
||||
|
||||
if (cookie->fq_domain)
|
||||
return 0;
|
||||
|
||||
atomic64_set(&cookie->fq_flush_start_cnt, 0);
|
||||
atomic64_set(&cookie->fq_flush_finish_cnt, 0);
|
||||
|
||||
queue = alloc_percpu(struct iova_fq);
|
||||
if (!queue) {
|
||||
pr_warn("iova flush queue initialization failed\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
struct iova_fq *fq = per_cpu_ptr(queue, cpu);
|
||||
|
||||
fq->head = 0;
|
||||
fq->tail = 0;
|
||||
|
||||
spin_lock_init(&fq->lock);
|
||||
|
||||
for (i = 0; i < IOVA_FQ_SIZE; i++)
|
||||
INIT_LIST_HEAD(&fq->entries[i].freelist);
|
||||
}
|
||||
|
||||
cookie->fq = queue;
|
||||
|
||||
timer_setup(&cookie->fq_timer, fq_flush_timeout, 0);
|
||||
atomic_set(&cookie->fq_timer_on, 0);
|
||||
/*
|
||||
* Prevent incomplete fq state being observable. Pairs with path from
|
||||
* __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova()
|
||||
*/
|
||||
smp_wmb();
|
||||
WRITE_ONCE(cookie->fq_domain, domain);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
|
||||
{
|
||||
if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
|
||||
@ -156,8 +357,10 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)
|
||||
if (!cookie)
|
||||
return;
|
||||
|
||||
if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule)
|
||||
if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
|
||||
iommu_dma_free_fq(cookie);
|
||||
put_iova_domain(&cookie->iovad);
|
||||
}
|
||||
|
||||
list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
|
||||
list_del(&msi->list);
|
||||
@ -294,17 +497,6 @@ static int iova_reserve_iommu_regions(struct device *dev,
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad)
|
||||
{
|
||||
struct iommu_dma_cookie *cookie;
|
||||
struct iommu_domain *domain;
|
||||
|
||||
cookie = container_of(iovad, struct iommu_dma_cookie, iovad);
|
||||
domain = cookie->fq_domain;
|
||||
|
||||
domain->ops->flush_iotlb_all(domain);
|
||||
}
|
||||
|
||||
static bool dev_is_untrusted(struct device *dev)
|
||||
{
|
||||
return dev_is_pci(dev) && to_pci_dev(dev)->untrusted;
|
||||
@ -315,30 +507,6 @@ static bool dev_use_swiotlb(struct device *dev)
|
||||
return IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev);
|
||||
}
|
||||
|
||||
/* sysfs updates are serialised by the mutex of the group owning @domain */
|
||||
int iommu_dma_init_fq(struct iommu_domain *domain)
|
||||
{
|
||||
struct iommu_dma_cookie *cookie = domain->iova_cookie;
|
||||
int ret;
|
||||
|
||||
if (cookie->fq_domain)
|
||||
return 0;
|
||||
|
||||
ret = init_iova_flush_queue(&cookie->iovad, iommu_dma_flush_iotlb_all,
|
||||
iommu_dma_entry_dtor);
|
||||
if (ret) {
|
||||
pr_warn("iova flush queue initialization failed\n");
|
||||
return ret;
|
||||
}
|
||||
/*
|
||||
* Prevent incomplete iovad->fq being observable. Pairs with path from
|
||||
* __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova()
|
||||
*/
|
||||
smp_wmb();
|
||||
WRITE_ONCE(cookie->fq_domain, domain);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* iommu_dma_init_domain - Initialise a DMA mapping domain
|
||||
* @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
|
||||
@ -442,14 +610,6 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
|
||||
|
||||
shift = iova_shift(iovad);
|
||||
iova_len = size >> shift;
|
||||
/*
|
||||
* Freeing non-power-of-two-sized allocations back into the IOVA caches
|
||||
* will come back to bite us badly, so we have to waste a bit of space
|
||||
* rounding up anything cacheable to make sure that can't happen. The
|
||||
* order of the unadjusted size will still match upon freeing.
|
||||
*/
|
||||
if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
|
||||
iova_len = roundup_pow_of_two(iova_len);
|
||||
|
||||
dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit);
|
||||
|
||||
@ -477,9 +637,9 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
|
||||
if (cookie->type == IOMMU_DMA_MSI_COOKIE)
|
||||
cookie->msi_iova -= size;
|
||||
else if (gather && gather->queued)
|
||||
queue_iova(iovad, iova_pfn(iovad, iova),
|
||||
queue_iova(cookie, iova_pfn(iovad, iova),
|
||||
size >> iova_shift(iovad),
|
||||
(unsigned long)gather->freelist);
|
||||
&gather->freelist);
|
||||
else
|
||||
free_iova_fast(iovad, iova_pfn(iovad, iova),
|
||||
size >> iova_shift(iovad));
|
||||
|
||||
@ -133,11 +133,6 @@ static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
|
||||
|
||||
/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
|
||||
are never going to work. */
|
||||
static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
|
||||
{
|
||||
return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
|
||||
}
|
||||
|
||||
static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
|
||||
{
|
||||
return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
|
||||
@ -1280,10 +1275,6 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain,
|
||||
unsigned long last_pfn,
|
||||
int retain_level)
|
||||
{
|
||||
BUG_ON(!domain_pfn_supported(domain, start_pfn));
|
||||
BUG_ON(!domain_pfn_supported(domain, last_pfn));
|
||||
BUG_ON(start_pfn > last_pfn);
|
||||
|
||||
dma_pte_clear_range(domain, start_pfn, last_pfn);
|
||||
|
||||
/* We don't need lock here; nobody else touches the iova range */
|
||||
@ -1303,35 +1294,30 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain,
|
||||
know the hardware page-walk will no longer touch them.
|
||||
The 'pte' argument is the *parent* PTE, pointing to the page that is to
|
||||
be freed. */
|
||||
static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
|
||||
int level, struct dma_pte *pte,
|
||||
struct page *freelist)
|
||||
static void dma_pte_list_pagetables(struct dmar_domain *domain,
|
||||
int level, struct dma_pte *pte,
|
||||
struct list_head *freelist)
|
||||
{
|
||||
struct page *pg;
|
||||
|
||||
pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
|
||||
pg->freelist = freelist;
|
||||
freelist = pg;
|
||||
list_add_tail(&pg->lru, freelist);
|
||||
|
||||
if (level == 1)
|
||||
return freelist;
|
||||
return;
|
||||
|
||||
pte = page_address(pg);
|
||||
do {
|
||||
if (dma_pte_present(pte) && !dma_pte_superpage(pte))
|
||||
freelist = dma_pte_list_pagetables(domain, level - 1,
|
||||
pte, freelist);
|
||||
dma_pte_list_pagetables(domain, level - 1, pte, freelist);
|
||||
pte++;
|
||||
} while (!first_pte_in_page(pte));
|
||||
|
||||
return freelist;
|
||||
}
|
||||
|
||||
static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
|
||||
struct dma_pte *pte, unsigned long pfn,
|
||||
unsigned long start_pfn,
|
||||
unsigned long last_pfn,
|
||||
struct page *freelist)
|
||||
static void dma_pte_clear_level(struct dmar_domain *domain, int level,
|
||||
struct dma_pte *pte, unsigned long pfn,
|
||||
unsigned long start_pfn, unsigned long last_pfn,
|
||||
struct list_head *freelist)
|
||||
{
|
||||
struct dma_pte *first_pte = NULL, *last_pte = NULL;
|
||||
|
||||
@ -1350,7 +1336,7 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
|
||||
/* These suborbinate page tables are going away entirely. Don't
|
||||
bother to clear them; we're just going to *free* them. */
|
||||
if (level > 1 && !dma_pte_superpage(pte))
|
||||
freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
|
||||
dma_pte_list_pagetables(domain, level - 1, pte, freelist);
|
||||
|
||||
dma_clear_pte(pte);
|
||||
if (!first_pte)
|
||||
@ -1358,10 +1344,10 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
|
||||
last_pte = pte;
|
||||
} else if (level > 1) {
|
||||
/* Recurse down into a level that isn't *entirely* obsolete */
|
||||
freelist = dma_pte_clear_level(domain, level - 1,
|
||||
phys_to_virt(dma_pte_addr(pte)),
|
||||
level_pfn, start_pfn, last_pfn,
|
||||
freelist);
|
||||
dma_pte_clear_level(domain, level - 1,
|
||||
phys_to_virt(dma_pte_addr(pte)),
|
||||
level_pfn, start_pfn, last_pfn,
|
||||
freelist);
|
||||
}
|
||||
next:
|
||||
pfn = level_pfn + level_size(level);
|
||||
@ -1370,47 +1356,28 @@ next:
|
||||
if (first_pte)
|
||||
domain_flush_cache(domain, first_pte,
|
||||
(void *)++last_pte - (void *)first_pte);
|
||||
|
||||
return freelist;
|
||||
}
|
||||
|
||||
/* We can't just free the pages because the IOMMU may still be walking
|
||||
the page tables, and may have cached the intermediate levels. The
|
||||
pages can only be freed after the IOTLB flush has been done. */
|
||||
static struct page *domain_unmap(struct dmar_domain *domain,
|
||||
unsigned long start_pfn,
|
||||
unsigned long last_pfn,
|
||||
struct page *freelist)
|
||||
static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn,
|
||||
unsigned long last_pfn, struct list_head *freelist)
|
||||
{
|
||||
BUG_ON(!domain_pfn_supported(domain, start_pfn));
|
||||
BUG_ON(!domain_pfn_supported(domain, last_pfn));
|
||||
BUG_ON(start_pfn > last_pfn);
|
||||
|
||||
/* we don't need lock here; nobody else touches the iova range */
|
||||
freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
|
||||
domain->pgd, 0, start_pfn, last_pfn,
|
||||
freelist);
|
||||
dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
|
||||
domain->pgd, 0, start_pfn, last_pfn, freelist);
|
||||
|
||||
/* free pgd */
|
||||
if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
|
||||
struct page *pgd_page = virt_to_page(domain->pgd);
|
||||
pgd_page->freelist = freelist;
|
||||
freelist = pgd_page;
|
||||
|
||||
list_add_tail(&pgd_page->lru, freelist);
|
||||
domain->pgd = NULL;
|
||||
}
|
||||
|
||||
return freelist;
|
||||
}
|
||||
|
||||
static void dma_free_pagelist(struct page *freelist)
|
||||
{
|
||||
struct page *pg;
|
||||
|
||||
while ((pg = freelist)) {
|
||||
freelist = pg->freelist;
|
||||
free_pgtable_page(page_address(pg));
|
||||
}
|
||||
}
|
||||
|
||||
/* iommu handling */
|
||||
@ -1878,17 +1845,16 @@ static void iommu_disable_translation(struct intel_iommu *iommu)
|
||||
|
||||
static int iommu_init_domains(struct intel_iommu *iommu)
|
||||
{
|
||||
u32 ndomains, nlongs;
|
||||
u32 ndomains;
|
||||
size_t size;
|
||||
|
||||
ndomains = cap_ndoms(iommu->cap);
|
||||
pr_debug("%s: Number of Domains supported <%d>\n",
|
||||
iommu->name, ndomains);
|
||||
nlongs = BITS_TO_LONGS(ndomains);
|
||||
|
||||
spin_lock_init(&iommu->lock);
|
||||
|
||||
iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
|
||||
iommu->domain_ids = bitmap_zalloc(ndomains, GFP_KERNEL);
|
||||
if (!iommu->domain_ids)
|
||||
return -ENOMEM;
|
||||
|
||||
@ -1903,7 +1869,7 @@ static int iommu_init_domains(struct intel_iommu *iommu)
|
||||
if (!iommu->domains || !iommu->domains[0]) {
|
||||
pr_err("%s: Allocating domain array failed\n",
|
||||
iommu->name);
|
||||
kfree(iommu->domain_ids);
|
||||
bitmap_free(iommu->domain_ids);
|
||||
kfree(iommu->domains);
|
||||
iommu->domain_ids = NULL;
|
||||
iommu->domains = NULL;
|
||||
@ -1964,7 +1930,7 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
|
||||
for (i = 0; i < elems; i++)
|
||||
kfree(iommu->domains[i]);
|
||||
kfree(iommu->domains);
|
||||
kfree(iommu->domain_ids);
|
||||
bitmap_free(iommu->domain_ids);
|
||||
iommu->domains = NULL;
|
||||
iommu->domain_ids = NULL;
|
||||
}
|
||||
@ -2095,11 +2061,10 @@ static void domain_exit(struct dmar_domain *domain)
|
||||
domain_remove_dev_info(domain);
|
||||
|
||||
if (domain->pgd) {
|
||||
struct page *freelist;
|
||||
LIST_HEAD(freelist);
|
||||
|
||||
freelist = domain_unmap(domain, 0,
|
||||
DOMAIN_MAX_PFN(domain->gaw), NULL);
|
||||
dma_free_pagelist(freelist);
|
||||
domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);
|
||||
put_pages_list(&freelist);
|
||||
}
|
||||
|
||||
free_domain_mem(domain);
|
||||
@ -2112,10 +2077,10 @@ static void domain_exit(struct dmar_domain *domain)
|
||||
*/
|
||||
static inline unsigned long context_get_sm_pds(struct pasid_table *table)
|
||||
{
|
||||
int pds, max_pde;
|
||||
unsigned long pds, max_pde;
|
||||
|
||||
max_pde = table->max_pasid >> PASID_PDE_SHIFT;
|
||||
pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS);
|
||||
pds = find_first_bit(&max_pde, MAX_NR_PASID_BITS);
|
||||
if (pds < 7)
|
||||
return 0;
|
||||
|
||||
@ -4192,19 +4157,17 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
|
||||
{
|
||||
struct dmar_drhd_unit *drhd;
|
||||
struct intel_iommu *iommu;
|
||||
struct page *freelist;
|
||||
LIST_HEAD(freelist);
|
||||
|
||||
freelist = domain_unmap(si_domain,
|
||||
start_vpfn, last_vpfn,
|
||||
NULL);
|
||||
domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist);
|
||||
|
||||
rcu_read_lock();
|
||||
for_each_active_iommu(iommu, drhd)
|
||||
iommu_flush_iotlb_psi(iommu, si_domain,
|
||||
start_vpfn, mhp->nr_pages,
|
||||
!freelist, 0);
|
||||
list_empty(&freelist), 0);
|
||||
rcu_read_unlock();
|
||||
dma_free_pagelist(freelist);
|
||||
put_pages_list(&freelist);
|
||||
}
|
||||
break;
|
||||
}
|
||||
@ -5211,8 +5174,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
|
||||
start_pfn = iova >> VTD_PAGE_SHIFT;
|
||||
last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
|
||||
|
||||
gather->freelist = domain_unmap(dmar_domain, start_pfn,
|
||||
last_pfn, gather->freelist);
|
||||
domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist);
|
||||
|
||||
if (dmar_domain->max_addr == iova + size)
|
||||
dmar_domain->max_addr = iova;
|
||||
@ -5248,9 +5210,10 @@ static void intel_iommu_tlb_sync(struct iommu_domain *domain,
|
||||
|
||||
for_each_domain_iommu(iommu_id, dmar_domain)
|
||||
iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
|
||||
start_pfn, nrpages, !gather->freelist, 0);
|
||||
start_pfn, nrpages,
|
||||
list_empty(&gather->freelist), 0);
|
||||
|
||||
dma_free_pagelist(gather->freelist);
|
||||
put_pages_list(&gather->freelist);
|
||||
}
|
||||
|
||||
static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
|
||||
|
||||
@ -246,13 +246,17 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp,
|
||||
__GFP_ZERO | ARM_V7S_TABLE_GFP_DMA, get_order(size));
|
||||
else if (lvl == 2)
|
||||
table = kmem_cache_zalloc(data->l2_tables, gfp);
|
||||
|
||||
if (!table)
|
||||
return NULL;
|
||||
|
||||
phys = virt_to_phys(table);
|
||||
if (phys != (arm_v7s_iopte)phys) {
|
||||
/* Doesn't fit in PTE */
|
||||
dev_err(dev, "Page table does not fit in PTE: %pa", &phys);
|
||||
goto out_free;
|
||||
}
|
||||
if (table && !cfg->coherent_walk) {
|
||||
if (!cfg->coherent_walk) {
|
||||
dma = dma_map_single(dev, table, size, DMA_TO_DEVICE);
|
||||
if (dma_mapping_error(dev, dma))
|
||||
goto out_free;
|
||||
|
||||
@ -315,11 +315,12 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
|
||||
static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,
|
||||
arm_lpae_iopte *ptep,
|
||||
arm_lpae_iopte curr,
|
||||
struct io_pgtable_cfg *cfg)
|
||||
struct arm_lpae_io_pgtable *data)
|
||||
{
|
||||
arm_lpae_iopte old, new;
|
||||
struct io_pgtable_cfg *cfg = &data->iop.cfg;
|
||||
|
||||
new = __pa(table) | ARM_LPAE_PTE_TYPE_TABLE;
|
||||
new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
|
||||
if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
|
||||
new |= ARM_LPAE_PTE_NSTABLE;
|
||||
|
||||
@ -380,7 +381,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
|
||||
if (!cptep)
|
||||
return -ENOMEM;
|
||||
|
||||
pte = arm_lpae_install_table(cptep, ptep, 0, cfg);
|
||||
pte = arm_lpae_install_table(cptep, ptep, 0, data);
|
||||
if (pte)
|
||||
__arm_lpae_free_pages(cptep, tblsz, cfg);
|
||||
} else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) {
|
||||
@ -592,7 +593,7 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
|
||||
__arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i]);
|
||||
}
|
||||
|
||||
pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg);
|
||||
pte = arm_lpae_install_table(tablep, ptep, blk_pte, data);
|
||||
if (pte != blk_pte) {
|
||||
__arm_lpae_free_pages(tablep, tablesz, cfg);
|
||||
/*
|
||||
|
||||
@ -288,11 +288,11 @@ int iommu_probe_device(struct device *dev)
|
||||
*/
|
||||
mutex_lock(&group->mutex);
|
||||
iommu_alloc_default_domain(group, dev);
|
||||
mutex_unlock(&group->mutex);
|
||||
|
||||
if (group->default_domain) {
|
||||
ret = __iommu_attach_device(group->default_domain, dev);
|
||||
if (ret) {
|
||||
mutex_unlock(&group->mutex);
|
||||
iommu_group_put(group);
|
||||
goto err_release;
|
||||
}
|
||||
@ -300,6 +300,7 @@ int iommu_probe_device(struct device *dev)
|
||||
|
||||
iommu_create_device_direct_mappings(group, dev);
|
||||
|
||||
mutex_unlock(&group->mutex);
|
||||
iommu_group_put(group);
|
||||
|
||||
if (ops->probe_finalize)
|
||||
|
||||
@ -24,8 +24,6 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad,
|
||||
static void init_iova_rcaches(struct iova_domain *iovad);
|
||||
static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
|
||||
static void free_iova_rcaches(struct iova_domain *iovad);
|
||||
static void fq_destroy_all_entries(struct iova_domain *iovad);
|
||||
static void fq_flush_timeout(struct timer_list *t);
|
||||
|
||||
static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node)
|
||||
{
|
||||
@ -63,8 +61,6 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule,
|
||||
iovad->start_pfn = start_pfn;
|
||||
iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad));
|
||||
iovad->max32_alloc_size = iovad->dma_32bit_pfn;
|
||||
iovad->flush_cb = NULL;
|
||||
iovad->fq = NULL;
|
||||
iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
|
||||
rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);
|
||||
rb_insert_color(&iovad->anchor.node, &iovad->rbroot);
|
||||
@ -73,62 +69,6 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule,
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(init_iova_domain);
|
||||
|
||||
static bool has_iova_flush_queue(struct iova_domain *iovad)
|
||||
{
|
||||
return !!iovad->fq;
|
||||
}
|
||||
|
||||
static void free_iova_flush_queue(struct iova_domain *iovad)
|
||||
{
|
||||
if (!has_iova_flush_queue(iovad))
|
||||
return;
|
||||
|
||||
if (timer_pending(&iovad->fq_timer))
|
||||
del_timer(&iovad->fq_timer);
|
||||
|
||||
fq_destroy_all_entries(iovad);
|
||||
|
||||
free_percpu(iovad->fq);
|
||||
|
||||
iovad->fq = NULL;
|
||||
iovad->flush_cb = NULL;
|
||||
iovad->entry_dtor = NULL;
|
||||
}
|
||||
|
||||
int init_iova_flush_queue(struct iova_domain *iovad,
|
||||
iova_flush_cb flush_cb, iova_entry_dtor entry_dtor)
|
||||
{
|
||||
struct iova_fq __percpu *queue;
|
||||
int cpu;
|
||||
|
||||
atomic64_set(&iovad->fq_flush_start_cnt, 0);
|
||||
atomic64_set(&iovad->fq_flush_finish_cnt, 0);
|
||||
|
||||
queue = alloc_percpu(struct iova_fq);
|
||||
if (!queue)
|
||||
return -ENOMEM;
|
||||
|
||||
iovad->flush_cb = flush_cb;
|
||||
iovad->entry_dtor = entry_dtor;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
struct iova_fq *fq;
|
||||
|
||||
fq = per_cpu_ptr(queue, cpu);
|
||||
fq->head = 0;
|
||||
fq->tail = 0;
|
||||
|
||||
spin_lock_init(&fq->lock);
|
||||
}
|
||||
|
||||
iovad->fq = queue;
|
||||
|
||||
timer_setup(&iovad->fq_timer, fq_flush_timeout, 0);
|
||||
atomic_set(&iovad->fq_timer_on, 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct rb_node *
|
||||
__get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn)
|
||||
{
|
||||
@ -497,6 +437,15 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
|
||||
unsigned long iova_pfn;
|
||||
struct iova *new_iova;
|
||||
|
||||
/*
|
||||
* Freeing non-power-of-two-sized allocations back into the IOVA caches
|
||||
* will come back to bite us badly, so we have to waste a bit of space
|
||||
* rounding up anything cacheable to make sure that can't happen. The
|
||||
* order of the unadjusted size will still match upon freeing.
|
||||
*/
|
||||
if (size < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
|
||||
size = roundup_pow_of_two(size);
|
||||
|
||||
iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1);
|
||||
if (iova_pfn)
|
||||
return iova_pfn;
|
||||
@ -539,144 +488,6 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(free_iova_fast);
|
||||
|
||||
#define fq_ring_for_each(i, fq) \
|
||||
for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
|
||||
|
||||
static inline bool fq_full(struct iova_fq *fq)
|
||||
{
|
||||
assert_spin_locked(&fq->lock);
|
||||
return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
|
||||
}
|
||||
|
||||
static inline unsigned fq_ring_add(struct iova_fq *fq)
|
||||
{
|
||||
unsigned idx = fq->tail;
|
||||
|
||||
assert_spin_locked(&fq->lock);
|
||||
|
||||
fq->tail = (idx + 1) % IOVA_FQ_SIZE;
|
||||
|
||||
return idx;
|
||||
}
|
||||
|
||||
static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)
|
||||
{
|
||||
u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt);
|
||||
unsigned idx;
|
||||
|
||||
assert_spin_locked(&fq->lock);
|
||||
|
||||
fq_ring_for_each(idx, fq) {
|
||||
|
||||
if (fq->entries[idx].counter >= counter)
|
||||
break;
|
||||
|
||||
if (iovad->entry_dtor)
|
||||
iovad->entry_dtor(fq->entries[idx].data);
|
||||
|
||||
free_iova_fast(iovad,
|
||||
fq->entries[idx].iova_pfn,
|
||||
fq->entries[idx].pages);
|
||||
|
||||
fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void iova_domain_flush(struct iova_domain *iovad)
|
||||
{
|
||||
atomic64_inc(&iovad->fq_flush_start_cnt);
|
||||
iovad->flush_cb(iovad);
|
||||
atomic64_inc(&iovad->fq_flush_finish_cnt);
|
||||
}
|
||||
|
||||
static void fq_destroy_all_entries(struct iova_domain *iovad)
|
||||
{
|
||||
int cpu;
|
||||
|
||||
/*
|
||||
* This code runs when the iova_domain is being detroyed, so don't
|
||||
* bother to free iovas, just call the entry_dtor on all remaining
|
||||
* entries.
|
||||
*/
|
||||
if (!iovad->entry_dtor)
|
||||
return;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);
|
||||
int idx;
|
||||
|
||||
fq_ring_for_each(idx, fq)
|
||||
iovad->entry_dtor(fq->entries[idx].data);
|
||||
}
|
||||
}
|
||||
|
||||
static void fq_flush_timeout(struct timer_list *t)
|
||||
{
|
||||
struct iova_domain *iovad = from_timer(iovad, t, fq_timer);
|
||||
int cpu;
|
||||
|
||||
atomic_set(&iovad->fq_timer_on, 0);
|
||||
iova_domain_flush(iovad);
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
unsigned long flags;
|
||||
struct iova_fq *fq;
|
||||
|
||||
fq = per_cpu_ptr(iovad->fq, cpu);
|
||||
spin_lock_irqsave(&fq->lock, flags);
|
||||
fq_ring_free(iovad, fq);
|
||||
spin_unlock_irqrestore(&fq->lock, flags);
|
||||
}
|
||||
}
|
||||
|
||||
void queue_iova(struct iova_domain *iovad,
|
||||
unsigned long pfn, unsigned long pages,
|
||||
unsigned long data)
|
||||
{
|
||||
struct iova_fq *fq;
|
||||
unsigned long flags;
|
||||
unsigned idx;
|
||||
|
||||
/*
|
||||
* Order against the IOMMU driver's pagetable update from unmapping
|
||||
* @pte, to guarantee that iova_domain_flush() observes that if called
|
||||
* from a different CPU before we release the lock below. Full barrier
|
||||
* so it also pairs with iommu_dma_init_fq() to avoid seeing partially
|
||||
* written fq state here.
|
||||
*/
|
||||
smp_mb();
|
||||
|
||||
fq = raw_cpu_ptr(iovad->fq);
|
||||
spin_lock_irqsave(&fq->lock, flags);
|
||||
|
||||
/*
|
||||
* First remove all entries from the flush queue that have already been
|
||||
* flushed out on another CPU. This makes the fq_full() check below less
|
||||
* likely to be true.
|
||||
*/
|
||||
fq_ring_free(iovad, fq);
|
||||
|
||||
if (fq_full(fq)) {
|
||||
iova_domain_flush(iovad);
|
||||
fq_ring_free(iovad, fq);
|
||||
}
|
||||
|
||||
idx = fq_ring_add(fq);
|
||||
|
||||
fq->entries[idx].iova_pfn = pfn;
|
||||
fq->entries[idx].pages = pages;
|
||||
fq->entries[idx].data = data;
|
||||
fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt);
|
||||
|
||||
spin_unlock_irqrestore(&fq->lock, flags);
|
||||
|
||||
/* Avoid false sharing as much as possible. */
|
||||
if (!atomic_read(&iovad->fq_timer_on) &&
|
||||
!atomic_xchg(&iovad->fq_timer_on, 1))
|
||||
mod_timer(&iovad->fq_timer,
|
||||
jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
|
||||
}
|
||||
|
||||
/**
|
||||
* put_iova_domain - destroys the iova domain
|
||||
* @iovad: - iova domain in question.
|
||||
@ -688,8 +499,6 @@ void put_iova_domain(struct iova_domain *iovad)
|
||||
|
||||
cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
|
||||
&iovad->cpuhp_dead);
|
||||
|
||||
free_iova_flush_queue(iovad);
|
||||
free_iova_rcaches(iovad);
|
||||
rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
|
||||
free_iova_mem(iova);
|
||||
|
||||
@ -71,6 +71,7 @@ struct viommu_domain {
|
||||
struct rb_root_cached mappings;
|
||||
|
||||
unsigned long nr_endpoints;
|
||||
bool bypass;
|
||||
};
|
||||
|
||||
struct viommu_endpoint {
|
||||
@ -310,8 +311,8 @@ out_unlock:
|
||||
*
|
||||
* On success, return the new mapping. Otherwise return NULL.
|
||||
*/
|
||||
static int viommu_add_mapping(struct viommu_domain *vdomain, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, u32 flags)
|
||||
static int viommu_add_mapping(struct viommu_domain *vdomain, u64 iova, u64 end,
|
||||
phys_addr_t paddr, u32 flags)
|
||||
{
|
||||
unsigned long irqflags;
|
||||
struct viommu_mapping *mapping;
|
||||
@ -322,7 +323,7 @@ static int viommu_add_mapping(struct viommu_domain *vdomain, unsigned long iova,
|
||||
|
||||
mapping->paddr = paddr;
|
||||
mapping->iova.start = iova;
|
||||
mapping->iova.last = iova + size - 1;
|
||||
mapping->iova.last = end;
|
||||
mapping->flags = flags;
|
||||
|
||||
spin_lock_irqsave(&vdomain->mappings_lock, irqflags);
|
||||
@ -337,26 +338,24 @@ static int viommu_add_mapping(struct viommu_domain *vdomain, unsigned long iova,
|
||||
*
|
||||
* @vdomain: the domain
|
||||
* @iova: start of the range
|
||||
* @size: size of the range. A size of 0 corresponds to the entire address
|
||||
* space.
|
||||
* @end: end of the range
|
||||
*
|
||||
* On success, returns the number of unmapped bytes (>= size)
|
||||
* On success, returns the number of unmapped bytes
|
||||
*/
|
||||
static size_t viommu_del_mappings(struct viommu_domain *vdomain,
|
||||
unsigned long iova, size_t size)
|
||||
u64 iova, u64 end)
|
||||
{
|
||||
size_t unmapped = 0;
|
||||
unsigned long flags;
|
||||
unsigned long last = iova + size - 1;
|
||||
struct viommu_mapping *mapping = NULL;
|
||||
struct interval_tree_node *node, *next;
|
||||
|
||||
spin_lock_irqsave(&vdomain->mappings_lock, flags);
|
||||
next = interval_tree_iter_first(&vdomain->mappings, iova, last);
|
||||
next = interval_tree_iter_first(&vdomain->mappings, iova, end);
|
||||
while (next) {
|
||||
node = next;
|
||||
mapping = container_of(node, struct viommu_mapping, iova);
|
||||
next = interval_tree_iter_next(node, iova, last);
|
||||
next = interval_tree_iter_next(node, iova, end);
|
||||
|
||||
/* Trying to split a mapping? */
|
||||
if (mapping->iova.start < iova)
|
||||
@ -376,6 +375,55 @@ static size_t viommu_del_mappings(struct viommu_domain *vdomain,
|
||||
return unmapped;
|
||||
}
|
||||
|
||||
/*
|
||||
* Fill the domain with identity mappings, skipping the device's reserved
|
||||
* regions.
|
||||
*/
|
||||
static int viommu_domain_map_identity(struct viommu_endpoint *vdev,
|
||||
struct viommu_domain *vdomain)
|
||||
{
|
||||
int ret;
|
||||
struct iommu_resv_region *resv;
|
||||
u64 iova = vdomain->domain.geometry.aperture_start;
|
||||
u64 limit = vdomain->domain.geometry.aperture_end;
|
||||
u32 flags = VIRTIO_IOMMU_MAP_F_READ | VIRTIO_IOMMU_MAP_F_WRITE;
|
||||
unsigned long granule = 1UL << __ffs(vdomain->domain.pgsize_bitmap);
|
||||
|
||||
iova = ALIGN(iova, granule);
|
||||
limit = ALIGN_DOWN(limit + 1, granule) - 1;
|
||||
|
||||
list_for_each_entry(resv, &vdev->resv_regions, list) {
|
||||
u64 resv_start = ALIGN_DOWN(resv->start, granule);
|
||||
u64 resv_end = ALIGN(resv->start + resv->length, granule) - 1;
|
||||
|
||||
if (resv_end < iova || resv_start > limit)
|
||||
/* No overlap */
|
||||
continue;
|
||||
|
||||
if (resv_start > iova) {
|
||||
ret = viommu_add_mapping(vdomain, iova, resv_start - 1,
|
||||
(phys_addr_t)iova, flags);
|
||||
if (ret)
|
||||
goto err_unmap;
|
||||
}
|
||||
|
||||
if (resv_end >= limit)
|
||||
return 0;
|
||||
|
||||
iova = resv_end + 1;
|
||||
}
|
||||
|
||||
ret = viommu_add_mapping(vdomain, iova, limit, (phys_addr_t)iova,
|
||||
flags);
|
||||
if (ret)
|
||||
goto err_unmap;
|
||||
return 0;
|
||||
|
||||
err_unmap:
|
||||
viommu_del_mappings(vdomain, 0, iova);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* viommu_replay_mappings - re-send MAP requests
|
||||
*
|
||||
@ -422,7 +470,7 @@ static int viommu_add_resv_mem(struct viommu_endpoint *vdev,
|
||||
size_t size;
|
||||
u64 start64, end64;
|
||||
phys_addr_t start, end;
|
||||
struct iommu_resv_region *region = NULL;
|
||||
struct iommu_resv_region *region = NULL, *next;
|
||||
unsigned long prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
|
||||
|
||||
start = start64 = le64_to_cpu(mem->start);
|
||||
@ -453,7 +501,12 @@ static int viommu_add_resv_mem(struct viommu_endpoint *vdev,
|
||||
if (!region)
|
||||
return -ENOMEM;
|
||||
|
||||
list_add(®ion->list, &vdev->resv_regions);
|
||||
/* Keep the list sorted */
|
||||
list_for_each_entry(next, &vdev->resv_regions, list) {
|
||||
if (next->start > region->start)
|
||||
break;
|
||||
}
|
||||
list_add_tail(®ion->list, &next->list);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -587,7 +640,9 @@ static struct iommu_domain *viommu_domain_alloc(unsigned type)
|
||||
{
|
||||
struct viommu_domain *vdomain;
|
||||
|
||||
if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
|
||||
if (type != IOMMU_DOMAIN_UNMANAGED &&
|
||||
type != IOMMU_DOMAIN_DMA &&
|
||||
type != IOMMU_DOMAIN_IDENTITY)
|
||||
return NULL;
|
||||
|
||||
vdomain = kzalloc(sizeof(*vdomain), GFP_KERNEL);
|
||||
@ -630,6 +685,21 @@ static int viommu_domain_finalise(struct viommu_endpoint *vdev,
|
||||
vdomain->map_flags = viommu->map_flags;
|
||||
vdomain->viommu = viommu;
|
||||
|
||||
if (domain->type == IOMMU_DOMAIN_IDENTITY) {
|
||||
if (virtio_has_feature(viommu->vdev,
|
||||
VIRTIO_IOMMU_F_BYPASS_CONFIG)) {
|
||||
vdomain->bypass = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
ret = viommu_domain_map_identity(vdev, vdomain);
|
||||
if (ret) {
|
||||
ida_free(&viommu->domain_ids, vdomain->id);
|
||||
vdomain->viommu = NULL;
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -637,8 +707,8 @@ static void viommu_domain_free(struct iommu_domain *domain)
|
||||
{
|
||||
struct viommu_domain *vdomain = to_viommu_domain(domain);
|
||||
|
||||
/* Free all remaining mappings (size 2^64) */
|
||||
viommu_del_mappings(vdomain, 0, 0);
|
||||
/* Free all remaining mappings */
|
||||
viommu_del_mappings(vdomain, 0, ULLONG_MAX);
|
||||
|
||||
if (vdomain->viommu)
|
||||
ida_free(&vdomain->viommu->domain_ids, vdomain->id);
|
||||
@ -673,7 +743,7 @@ static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev)
|
||||
|
||||
/*
|
||||
* In the virtio-iommu device, when attaching the endpoint to a new
|
||||
* domain, it is detached from the old one and, if as as a result the
|
||||
* domain, it is detached from the old one and, if as a result the
|
||||
* old domain isn't attached to any endpoint, all mappings are removed
|
||||
* from the old domain and it is freed.
|
||||
*
|
||||
@ -691,6 +761,9 @@ static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev)
|
||||
.domain = cpu_to_le32(vdomain->id),
|
||||
};
|
||||
|
||||
if (vdomain->bypass)
|
||||
req.flags |= cpu_to_le32(VIRTIO_IOMMU_ATTACH_F_BYPASS);
|
||||
|
||||
for (i = 0; i < fwspec->num_ids; i++) {
|
||||
req.endpoint = cpu_to_le32(fwspec->ids[i]);
|
||||
|
||||
@ -720,6 +793,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova,
|
||||
{
|
||||
int ret;
|
||||
u32 flags;
|
||||
u64 end = iova + size - 1;
|
||||
struct virtio_iommu_req_map map;
|
||||
struct viommu_domain *vdomain = to_viommu_domain(domain);
|
||||
|
||||
@ -730,7 +804,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova,
|
||||
if (flags & ~vdomain->map_flags)
|
||||
return -EINVAL;
|
||||
|
||||
ret = viommu_add_mapping(vdomain, iova, paddr, size, flags);
|
||||
ret = viommu_add_mapping(vdomain, iova, end, paddr, flags);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
@ -739,7 +813,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova,
|
||||
.domain = cpu_to_le32(vdomain->id),
|
||||
.virt_start = cpu_to_le64(iova),
|
||||
.phys_start = cpu_to_le64(paddr),
|
||||
.virt_end = cpu_to_le64(iova + size - 1),
|
||||
.virt_end = cpu_to_le64(end),
|
||||
.flags = cpu_to_le32(flags),
|
||||
};
|
||||
|
||||
@ -748,7 +822,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova,
|
||||
|
||||
ret = viommu_send_req_sync(vdomain->viommu, &map, sizeof(map));
|
||||
if (ret)
|
||||
viommu_del_mappings(vdomain, iova, size);
|
||||
viommu_del_mappings(vdomain, iova, end);
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -761,7 +835,7 @@ static size_t viommu_unmap(struct iommu_domain *domain, unsigned long iova,
|
||||
struct virtio_iommu_req_unmap unmap;
|
||||
struct viommu_domain *vdomain = to_viommu_domain(domain);
|
||||
|
||||
unmapped = viommu_del_mappings(vdomain, iova, size);
|
||||
unmapped = viommu_del_mappings(vdomain, iova, iova + size - 1);
|
||||
if (unmapped < size)
|
||||
return 0;
|
||||
|
||||
@ -1132,6 +1206,7 @@ static unsigned int features[] = {
|
||||
VIRTIO_IOMMU_F_DOMAIN_RANGE,
|
||||
VIRTIO_IOMMU_F_PROBE,
|
||||
VIRTIO_IOMMU_F_MMIO,
|
||||
VIRTIO_IOMMU_F_BYPASS_CONFIG,
|
||||
};
|
||||
|
||||
static struct virtio_device_id id_table[] = {
|
||||
|
||||
@ -292,14 +292,6 @@ vduse_domain_alloc_iova(struct iova_domain *iovad,
|
||||
unsigned long iova_len = iova_align(iovad, size) >> shift;
|
||||
unsigned long iova_pfn;
|
||||
|
||||
/*
|
||||
* Freeing non-power-of-two-sized allocations back into the IOVA caches
|
||||
* will come back to bite us badly, so we have to waste a bit of space
|
||||
* rounding up anything cacheable to make sure that can't happen. The
|
||||
* order of the unadjusted size will still match upon freeing.
|
||||
*/
|
||||
if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
|
||||
iova_len = roundup_pow_of_two(iova_len);
|
||||
iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true);
|
||||
|
||||
return iova_pfn << shift;
|
||||
|
||||
@ -8,12 +8,6 @@
|
||||
#ifndef __INTEL_SVM_H__
|
||||
#define __INTEL_SVM_H__
|
||||
|
||||
/* Values for rxwp in fault_cb callback */
|
||||
#define SVM_REQ_READ (1<<3)
|
||||
#define SVM_REQ_WRITE (1<<2)
|
||||
#define SVM_REQ_EXEC (1<<1)
|
||||
#define SVM_REQ_PRIV (1<<0)
|
||||
|
||||
/* Page Request Queue depth */
|
||||
#define PRQ_ORDER 2
|
||||
#define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20)
|
||||
|
||||
@ -186,7 +186,7 @@ struct iommu_iotlb_gather {
|
||||
unsigned long start;
|
||||
unsigned long end;
|
||||
size_t pgsize;
|
||||
struct page *freelist;
|
||||
struct list_head freelist;
|
||||
bool queued;
|
||||
};
|
||||
|
||||
@ -399,6 +399,7 @@ static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
|
||||
{
|
||||
*gather = (struct iommu_iotlb_gather) {
|
||||
.start = ULONG_MAX,
|
||||
.freelist = LIST_HEAD_INIT(gather->freelist),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -12,7 +12,6 @@
|
||||
#include <linux/types.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/rbtree.h>
|
||||
#include <linux/atomic.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
|
||||
/* iova structure */
|
||||
@ -35,35 +34,6 @@ struct iova_rcache {
|
||||
struct iova_cpu_rcache __percpu *cpu_rcaches;
|
||||
};
|
||||
|
||||
struct iova_domain;
|
||||
|
||||
/* Call-Back from IOVA code into IOMMU drivers */
|
||||
typedef void (* iova_flush_cb)(struct iova_domain *domain);
|
||||
|
||||
/* Destructor for per-entry data */
|
||||
typedef void (* iova_entry_dtor)(unsigned long data);
|
||||
|
||||
/* Number of entries per Flush Queue */
|
||||
#define IOVA_FQ_SIZE 256
|
||||
|
||||
/* Timeout (in ms) after which entries are flushed from the Flush-Queue */
|
||||
#define IOVA_FQ_TIMEOUT 10
|
||||
|
||||
/* Flush Queue entry for defered flushing */
|
||||
struct iova_fq_entry {
|
||||
unsigned long iova_pfn;
|
||||
unsigned long pages;
|
||||
unsigned long data;
|
||||
u64 counter; /* Flush counter when this entrie was added */
|
||||
};
|
||||
|
||||
/* Per-CPU Flush Queue structure */
|
||||
struct iova_fq {
|
||||
struct iova_fq_entry entries[IOVA_FQ_SIZE];
|
||||
unsigned head, tail;
|
||||
spinlock_t lock;
|
||||
};
|
||||
|
||||
/* holds all the iova translations for a domain */
|
||||
struct iova_domain {
|
||||
spinlock_t iova_rbtree_lock; /* Lock to protect update of rbtree */
|
||||
@ -74,27 +44,9 @@ struct iova_domain {
|
||||
unsigned long start_pfn; /* Lower limit for this domain */
|
||||
unsigned long dma_32bit_pfn;
|
||||
unsigned long max32_alloc_size; /* Size of last failed allocation */
|
||||
struct iova_fq __percpu *fq; /* Flush Queue */
|
||||
|
||||
atomic64_t fq_flush_start_cnt; /* Number of TLB flushes that
|
||||
have been started */
|
||||
|
||||
atomic64_t fq_flush_finish_cnt; /* Number of TLB flushes that
|
||||
have been finished */
|
||||
|
||||
struct iova anchor; /* rbtree lookup anchor */
|
||||
|
||||
struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
|
||||
|
||||
iova_flush_cb flush_cb; /* Call-Back function to flush IOMMU
|
||||
TLBs */
|
||||
|
||||
iova_entry_dtor entry_dtor; /* IOMMU driver specific destructor for
|
||||
iova entry */
|
||||
|
||||
struct timer_list fq_timer; /* Timer to regularily empty the
|
||||
flush-queues */
|
||||
atomic_t fq_timer_on; /* 1 when timer is active, 0
|
||||
when not */
|
||||
struct hlist_node cpuhp_dead;
|
||||
};
|
||||
|
||||
@ -144,17 +96,12 @@ struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
|
||||
bool size_aligned);
|
||||
void free_iova_fast(struct iova_domain *iovad, unsigned long pfn,
|
||||
unsigned long size);
|
||||
void queue_iova(struct iova_domain *iovad,
|
||||
unsigned long pfn, unsigned long pages,
|
||||
unsigned long data);
|
||||
unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
|
||||
unsigned long limit_pfn, bool flush_rcache);
|
||||
struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
|
||||
unsigned long pfn_hi);
|
||||
void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
|
||||
unsigned long start_pfn);
|
||||
int init_iova_flush_queue(struct iova_domain *iovad,
|
||||
iova_flush_cb flush_cb, iova_entry_dtor entry_dtor);
|
||||
struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
|
||||
void put_iova_domain(struct iova_domain *iovad);
|
||||
#else
|
||||
@ -189,12 +136,6 @@ static inline void free_iova_fast(struct iova_domain *iovad,
|
||||
{
|
||||
}
|
||||
|
||||
static inline void queue_iova(struct iova_domain *iovad,
|
||||
unsigned long pfn, unsigned long pages,
|
||||
unsigned long data)
|
||||
{
|
||||
}
|
||||
|
||||
static inline unsigned long alloc_iova_fast(struct iova_domain *iovad,
|
||||
unsigned long size,
|
||||
unsigned long limit_pfn,
|
||||
@ -216,13 +157,6 @@ static inline void init_iova_domain(struct iova_domain *iovad,
|
||||
{
|
||||
}
|
||||
|
||||
static inline int init_iova_flush_queue(struct iova_domain *iovad,
|
||||
iova_flush_cb flush_cb,
|
||||
iova_entry_dtor entry_dtor)
|
||||
{
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
static inline struct iova *find_iova(struct iova_domain *iovad,
|
||||
unsigned long pfn)
|
||||
{
|
||||
|
||||
@ -101,8 +101,9 @@ TRACE_EVENT(map,
|
||||
__entry->size = size;
|
||||
),
|
||||
|
||||
TP_printk("IOMMU: iova=0x%016llx paddr=0x%016llx size=%zu",
|
||||
__entry->iova, __entry->paddr, __entry->size
|
||||
TP_printk("IOMMU: iova=0x%016llx - 0x%016llx paddr=0x%016llx size=%zu",
|
||||
__entry->iova, __entry->iova + __entry->size, __entry->paddr,
|
||||
__entry->size
|
||||
)
|
||||
);
|
||||
|
||||
@ -124,8 +125,9 @@ TRACE_EVENT(unmap,
|
||||
__entry->unmapped_size = unmapped_size;
|
||||
),
|
||||
|
||||
TP_printk("IOMMU: iova=0x%016llx size=%zu unmapped_size=%zu",
|
||||
__entry->iova, __entry->size, __entry->unmapped_size
|
||||
TP_printk("IOMMU: iova=0x%016llx - 0x%016llx size=%zu unmapped_size=%zu",
|
||||
__entry->iova, __entry->iova + __entry->size,
|
||||
__entry->size, __entry->unmapped_size
|
||||
)
|
||||
);
|
||||
|
||||
|
||||
@ -16,6 +16,7 @@
|
||||
#define VIRTIO_IOMMU_F_BYPASS 3
|
||||
#define VIRTIO_IOMMU_F_PROBE 4
|
||||
#define VIRTIO_IOMMU_F_MMIO 5
|
||||
#define VIRTIO_IOMMU_F_BYPASS_CONFIG 6
|
||||
|
||||
struct virtio_iommu_range_64 {
|
||||
__le64 start;
|
||||
@ -36,6 +37,8 @@ struct virtio_iommu_config {
|
||||
struct virtio_iommu_range_32 domain_range;
|
||||
/* Probe buffer size */
|
||||
__le32 probe_size;
|
||||
__u8 bypass;
|
||||
__u8 reserved[3];
|
||||
};
|
||||
|
||||
/* Request types */
|
||||
@ -66,11 +69,14 @@ struct virtio_iommu_req_tail {
|
||||
__u8 reserved[3];
|
||||
};
|
||||
|
||||
#define VIRTIO_IOMMU_ATTACH_F_BYPASS (1 << 0)
|
||||
|
||||
struct virtio_iommu_req_attach {
|
||||
struct virtio_iommu_req_head head;
|
||||
__le32 domain;
|
||||
__le32 endpoint;
|
||||
__u8 reserved[8];
|
||||
__le32 flags;
|
||||
__u8 reserved[4];
|
||||
struct virtio_iommu_req_tail tail;
|
||||
};
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user