Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-25 20:32:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

accel/ivpu: Rename and cleanup MMU600 page tables

Browse Source 
Simplify and unify naming convention in MMU600 page tables
configuration.

All DMA addresses in page tables directly accessed by VPU are called
with _dma sufix and all CPU pointers to those page tables have _ptr
sufix.

Base pointers used to do a page walk on the CPU have corresponding
names:

 pud_ptrs (pointers used to get access to PUD DMA)
 pmd_ptrs (pointers used to get access to PMD DMA)
 pte_ptrs (pointers used to get access to PTE DMA)

with the following convention:

 u64 *pud_dma_ptr = pud_ptrs[pgd_idx];
 *pud_dma_ptr = pud_dma;

 u64 *pmd_dma_ptr = pmd_ptrs[pgd_idx][pud_idx];
 *pmd_dma_ptr = pmd_dma;

 u64 *pte_dma_ptr = pte_ptrs[pgd_idx][pud_idx][pmd_idx];
 *pte_dma_ptr = pte_dma;

On the way change to coherent dma allocation, _wc is only valid on ARM
and was used by mistake.

Signed-off-by: Karol Wachowski <karol.wachowski@linux.intel.com>
Reviewed-by: Stanislaw Gruszka <stanislaw.gruszka@linux.intel.com>
Reviewed-by: Jeffrey Hugo <quic_jhugo@quicinc.com>
Signed-off-by: Stanislaw Gruszka <stanislaw.gruszka@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20230518131605.650622-5-stanislaw.gruszka@linux.intel.com
This commit is contained in:
Karol Wachowski 2023-05-18 15:16:04 +02:00 committed by Stanislaw Gruszka
parent a4172d6cf0
commit 103d2ea139
2 changed files with 97 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

181
drivers/accel/ivpu/ivpu_mmu_context.c
View File

@ -39,123 +39,125 @@
static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
{
dma_addr_t pgd_dma;
u64 *pgd;
pgd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma, GFP_KERNEL);
if (!pgd)
pgtable->pgd_dma_ptr = dma_alloc_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma,
GFP_KERNEL);
if (!pgtable->pgd_dma_ptr)
return -ENOMEM;
pgtable->pgd = pgd;
pgtable->pgd_dma = pgd_dma;
return 0;
}
static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)
{
if (cpu_addr)
dma_free_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, cpu_addr,
dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK);
}
static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
{
int pgd_idx, pud_idx, pmd_idx;
dma_addr_t pud_dma, pmd_dma, pte_dma;
u64 *pud_dma_ptr, *pmd_dma_ptr, *pte_dma_ptr;
for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) {
u64 **pud_entries = pgtable->pgd_cpu_entries[pgd_idx];
u64 *pud = pgtable->pgd_entries[pgd_idx];
pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
pud_dma = pgtable->pgd_dma_ptr[pgd_idx];
if (!pud_entries)
if (!pud_dma_ptr)
continue;
for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) {
u64 **pmd_entries = pgtable->pgd_far_entries[pgd_idx][pud_idx];
u64 *pmd = pgtable->pgd_cpu_entries[pgd_idx][pud_idx];
pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];
if (!pmd_entries)
if (!pmd_dma_ptr)
continue;
for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) {
if (pmd_entries[pmd_idx])
dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE,
pmd_entries[pmd_idx],
pmd[pmd_idx] & ~IVPU_MMU_ENTRY_FLAGS_MASK);
pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];
ivpu_mmu_pgtable_free(vdev, pte_dma_ptr, pte_dma);
}
kfree(pmd_entries);
dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE,
pud_entries[pud_idx],
pud[pud_idx] & ~IVPU_MMU_ENTRY_FLAGS_MASK);
kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);
ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
}
kfree(pud_entries);
dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd_entries[pgd_idx],
pgtable->pgd[pgd_idx] & ~IVPU_MMU_ENTRY_FLAGS_MASK);
kfree(pgtable->pmd_ptrs[pgd_idx]);
kfree(pgtable->pte_ptrs[pgd_idx]);
ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
}
dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd,
pgtable->pgd_dma & ~IVPU_MMU_ENTRY_FLAGS_MASK);
ivpu_mmu_pgtable_free(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);
}
static u64*
ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx)
{
u64 ***far_pud_entries;
u64 **pud_entries;
u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
dma_addr_t pud_dma;
u64 *pud;
if (pgtable->pgd_entries[pgd_idx])
return pgtable->pgd_entries[pgd_idx];
if (pud_dma_ptr)
return pud_dma_ptr;
pud = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL);
if (!pud)
pud_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL);
if (!pud_dma_ptr)
return NULL;
pud_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
if (!pud_entries)
goto err_free_pud;
drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]);
pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
if (!pgtable->pmd_ptrs[pgd_idx])
goto err_free_pud_dma_ptr;
far_pud_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
if (!far_pud_entries)
goto err_free_pud_entries;
drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]);
pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
if (!pgtable->pte_ptrs[pgd_idx])
goto err_free_pmd_ptrs;
pgtable->pgd[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;
pgtable->pgd_entries[pgd_idx] = pud;
pgtable->pgd_cpu_entries[pgd_idx] = pud_entries;
pgtable->pgd_far_entries[pgd_idx] = far_pud_entries;
pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr;
pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;
return pud;
return pud_dma_ptr;
err_free_pud_entries:
kfree(pud_entries);
err_free_pmd_ptrs:
kfree(pgtable->pmd_ptrs[pgd_idx]);
err_free_pud:
dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pud, pud_dma);
err_free_pud_dma_ptr:
ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
return NULL;
}
static u64*
ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
int pgd_idx, int pud_idx)
ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx,
int pud_idx)
{
u64 **pmd_entries;
u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
dma_addr_t pmd_dma;
u64 *pmd;
if (pgtable->pgd_cpu_entries[pgd_idx][pud_idx])
return pgtable->pgd_cpu_entries[pgd_idx][pud_idx];
if (pmd_dma_ptr)
return pmd_dma_ptr;
pmd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL);
if (!pmd)
pmd_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL);
if (!pmd_dma_ptr)
return NULL;
pmd_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
if (!pmd_entries)
goto err_free_pmd;
drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);
pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
if (!pgtable->pte_ptrs[pgd_idx][pud_idx])
goto err_free_pmd_dma_ptr;
pgtable->pgd_entries[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;
pgtable->pgd_cpu_entries[pgd_idx][pud_idx] = pmd;
pgtable->pgd_far_entries[pgd_idx][pud_idx] = pmd_entries;
pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;
pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;
return pmd;
return pmd_dma_ptr;
err_free_pmd:
dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pmd, pmd_dma);
err_free_pmd_dma_ptr:
ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
return NULL;
}
@ -163,20 +165,20 @@ static u64*
ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
int pgd_idx, int pud_idx, int pmd_idx)
{
u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
dma_addr_t pte_dma;
u64 *pte;
if (pgtable->pgd_far_entries[pgd_idx][pud_idx][pmd_idx])
return pgtable->pgd_far_entries[pgd_idx][pud_idx][pmd_idx];
if (pte_dma_ptr)
return pte_dma_ptr;
pte = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL);
if (!pte)
pte_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL);
if (!pte_dma_ptr)
return NULL;
pgtable->pgd_cpu_entries[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;
pgtable->pgd_far_entries[pgd_idx][pud_idx][pmd_idx] = pte;
pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;
pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;
return pte;
return pte_dma_ptr;
}
static int
@ -189,20 +191,20 @@ ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx
int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
/* Allocate PUD - first level page table if needed */
/* Allocate PUD - second level page table if needed */
if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx))
return -ENOMEM;
/* Allocate PMD - second level page table if needed */
/* Allocate PMD - third level page table if needed */
if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx))
return -ENOMEM;
/* Allocate PTE - third level page table if needed */
/* Allocate PTE - fourth level page table if needed */
pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);
if (!pte)
return -ENOMEM;
/* Update PTE - third level page table with DMA address */
/* Update PTE */
pte[pte_idx] = dma_addr | prot;
return 0;
@ -216,40 +218,39 @@ static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_ad
int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
/* Update PTE with dummy physical address and clear flags */
ctx->pgtable.pgd_far_entries[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;
ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;
}
static void
ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
{
struct ivpu_mmu_pgtable *pgtable = &ctx->pgtable;
u64 end_addr = vpu_addr + size;
u64 *pgd = ctx->pgtable.pgd;
/* Align to PMD entry (2 MB) */
vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1);
while (vpu_addr < end_addr) {
int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
u64 pud_end = (pgd_idx + 1) * (u64)IVPU_MMU_PUD_MAP_SIZE;
u64 *pud = ctx->pgtable.pgd_entries[pgd_idx];
while (vpu_addr < end_addr && vpu_addr < pud_end) {
int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
u64 pmd_end = (pud_idx + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE;
u64 *pmd = ctx->pgtable.pgd_cpu_entries[pgd_idx][pud_idx];
while (vpu_addr < end_addr && vpu_addr < pmd_end) {
int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
u64 *pte = ctx->pgtable.pgd_far_entries
[pgd_idx][pud_idx][pmd_idx];
clflush_cache_range(pte, IVPU_MMU_PGTABLE_SIZE);
clflush_cache_range(pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx],
IVPU_MMU_PGTABLE_SIZE);
vpu_addr += IVPU_MMU_PTE_MAP_SIZE;
}
clflush_cache_range(pmd, IVPU_MMU_PGTABLE_SIZE);
clflush_cache_range(pgtable->pmd_ptrs[pgd_idx][pud_idx],
IVPU_MMU_PGTABLE_SIZE);
}
clflush_cache_range(pud, IVPU_MMU_PGTABLE_SIZE);
clflush_cache_range(pgtable->pud_ptrs[pgd_idx], IVPU_MMU_PGTABLE_SIZE);
}
clflush_cache_range(pgd, IVPU_MMU_PGTABLE_SIZE);
clflush_cache_range(pgtable->pgd_dma_ptr, IVPU_MMU_PGTABLE_SIZE);
}
static int
@ -305,7 +306,7 @@ ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
mutex_lock(&ctx->lock);
for_each_sgtable_dma_sg(sgt, sg, i) {
u64 dma_addr = sg_dma_address(sg) - sg->offset;
dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset;
size_t size = sg_dma_len(sg) + sg->offset;
ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);
@ -402,11 +403,15 @@ ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u3
static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
{
drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd);
if (drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd_dma_ptr))
return;
mutex_destroy(&ctx->lock);
ivpu_mmu_pgtable_free(vdev, &ctx->pgtable);
ivpu_mmu_pgtables_free(vdev, &ctx->pgtable);
drm_mm_takedown(&ctx->mm);
ctx->pgtable.pgd_dma_ptr = NULL;
ctx->pgtable.pgd_dma = 0;
}
int ivpu_mmu_global_context_init(struct ivpu_device *vdev)

8
drivers/accel/ivpu/ivpu_mmu_context.h
View File

@ -15,10 +15,10 @@ struct ivpu_addr_range;
#define IVPU_MMU_PGTABLE_ENTRIES 512ull
struct ivpu_mmu_pgtable {
u64 ***pgd_far_entries[IVPU_MMU_PGTABLE_ENTRIES];
u64 **pgd_cpu_entries[IVPU_MMU_PGTABLE_ENTRIES];
u64 *pgd_entries[IVPU_MMU_PGTABLE_ENTRIES];
u64 *pgd;
u64 ***pte_ptrs[IVPU_MMU_PGTABLE_ENTRIES];
u64 **pmd_ptrs[IVPU_MMU_PGTABLE_ENTRIES];
u64 *pud_ptrs[IVPU_MMU_PGTABLE_ENTRIES];
u64 *pgd_dma_ptr;
dma_addr_t pgd_dma;
};