ANDROID: GKI: iommu: dma-mapping-fast: Fast ARMv7/v8 Long Descriptor Format
Snapshot of vendor added support for Fast ARMv7/v8 Long Descriptor Format. Signed-off-by: Charan Teja Reddy <charante@codeaurora.org> Signed-off-by: Liam Mark <lmark@codeaurora.org> Signed-off-by: Patrick Daly <pdaly@codeaurora.org> Signed-off-by: Prakash Gupta <guptap@codeaurora.org> Signed-off-by: Qingqing Zhou <qqzhou@codeaurora.org> Signed-off-by: Rishabh Bhatnagar <rishabhb@codeaurora.org> Signed-off-by: Shiraz Hashim <shashim@codeaurora.org> Signed-off-by: Sudarshan Rajagopalan <sudaraja@codeaurora.org> Signed-off-by: Swathi Sridhar <swatsrid@codeaurora.org> Signed-off-by: Vinayak Menon <vinmenon@codeaurora.org> Bug: 155522481 Signed-off-by: Mark Salyzyn <salyzyn@google.com> [saravanak snapshot from commit 79efc458af96 and disabled for ARM] Signed-off-by: Saravana Kannan <saravanak@google.com> Change-Id: Ifae8f889737f25cd8a9729101cd8251974109842
This commit is contained in:
Swathi Sridhar
committed by
Saravana Kannan
Saravana Kannan
parent
0f671ed83a
commit
9b76f03f94
@@ -23,6 +23,8 @@ struct dma_iommu_mapping {
|
||||
void *bitmap;
|
||||
size_t bits;
|
||||
dma_addr_t base;
|
||||
|
||||
struct dma_fast_smmu_mapping *fast;
|
||||
};
|
||||
|
||||
#ifdef CONFIG_ARM64_DMA_USE_IOMMU
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/tlbflush.h>
|
||||
#include <asm/dma-iommu.h>
|
||||
#include <linux/dma-mapping-fast.h>
|
||||
|
||||
static int swiotlb __ro_after_init;
|
||||
|
||||
@@ -1009,16 +1010,19 @@ iommu_init_mapping(struct device *dev, struct dma_iommu_mapping *mapping)
|
||||
static int arm_iommu_get_dma_cookie(struct device *dev,
|
||||
struct dma_iommu_mapping *mapping)
|
||||
{
|
||||
int s1_bypass = 0;
|
||||
int s1_bypass = 0, is_fast = 0;
|
||||
int err = 0;
|
||||
|
||||
mutex_lock(&iommu_dma_init_mutex);
|
||||
|
||||
iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_S1_BYPASS,
|
||||
&s1_bypass);
|
||||
iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_FAST, &is_fast);
|
||||
|
||||
if (s1_bypass)
|
||||
mapping->ops = &arm64_swiotlb_dma_ops;
|
||||
else if (is_fast)
|
||||
err = fast_smmu_init_mapping(dev, mapping);
|
||||
else
|
||||
err = iommu_init_mapping(dev, mapping);
|
||||
|
||||
|
||||
@@ -58,6 +58,18 @@ config IOMMU_IO_PGTABLE_ARMV7S_SELFTEST
|
||||
|
||||
If unsure, say N here.
|
||||
|
||||
config IOMMU_IO_PGTABLE_FAST
|
||||
bool "Fast ARMv7/v8 Long Descriptor Format"
|
||||
depends on ARM64_DMA_USE_IOMMU && IOMMU_DMA
|
||||
help
|
||||
Enable support for a subset of the ARM long descriptor pagetable
|
||||
format. This allocator achieves fast performance by
|
||||
pre-allocating and pre-populating page table memory up front.
|
||||
only supports a 32 bit virtual address space.
|
||||
|
||||
This implementation is mainly optimized for use cases where the
|
||||
buffers are small (<= 64K) since it only supports 4K page sizes.
|
||||
|
||||
endmenu
|
||||
|
||||
config IOMMU_DEBUGFS
|
||||
|
||||
@@ -8,6 +8,7 @@ obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o
|
||||
obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o
|
||||
obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o
|
||||
obj-$(CONFIG_IOMMU_IOVA) += iova.o
|
||||
obj-$(CONFIG_IOMMU_IO_PGTABLE_FAST) += io-pgtable-fast.o dma-mapping-fast.o
|
||||
obj-$(CONFIG_OF_IOMMU) += of_iommu.o
|
||||
obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o
|
||||
obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o amd_iommu_quirks.o
|
||||
|
||||
1249
drivers/iommu/dma-mapping-fast.c
Normal file
1249
drivers/iommu/dma-mapping-fast.c
Normal file
File diff suppressed because it is too large
Load Diff
816
drivers/iommu/io-pgtable-fast.c
Normal file
816
drivers/iommu/io-pgtable-fast.c
Normal file
@@ -0,0 +1,816 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2016-2017, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#define pr_fmt(fmt) "io-pgtable-fast: " fmt
|
||||
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/io-pgtable.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/sizes.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/io-pgtable-fast.h>
|
||||
#include <linux/mm.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <linux/vmalloc.h>
|
||||
|
||||
#define AV8L_FAST_MAX_ADDR_BITS 48
|
||||
|
||||
/* Struct accessors */
|
||||
#define iof_pgtable_to_data(x) \
|
||||
container_of((x), struct av8l_fast_io_pgtable, iop)
|
||||
|
||||
#define iof_pgtable_ops_to_pgtable(x) \
|
||||
container_of((x), struct io_pgtable, ops)
|
||||
|
||||
#define iof_pgtable_ops_to_data(x) \
|
||||
iof_pgtable_to_data(iof_pgtable_ops_to_pgtable(x))
|
||||
|
||||
struct av8l_fast_io_pgtable {
|
||||
struct io_pgtable iop;
|
||||
av8l_fast_iopte *pgd;
|
||||
av8l_fast_iopte *puds[4];
|
||||
av8l_fast_iopte *pmds;
|
||||
struct page **pages; /* page table memory */
|
||||
int nr_pages;
|
||||
dma_addr_t base;
|
||||
dma_addr_t start;
|
||||
dma_addr_t end;
|
||||
};
|
||||
|
||||
/* Page table bits */
|
||||
#define AV8L_FAST_PTE_TYPE_SHIFT 0
|
||||
#define AV8L_FAST_PTE_TYPE_MASK 0x3
|
||||
|
||||
#define AV8L_FAST_PTE_TYPE_BLOCK 1
|
||||
#define AV8L_FAST_PTE_TYPE_TABLE 3
|
||||
#define AV8L_FAST_PTE_TYPE_PAGE 3
|
||||
|
||||
#define AV8L_FAST_PTE_NSTABLE (((av8l_fast_iopte)1) << 63)
|
||||
#define AV8L_FAST_PTE_XN (((av8l_fast_iopte)3) << 53)
|
||||
#define AV8L_FAST_PTE_AF (((av8l_fast_iopte)1) << 10)
|
||||
#define AV8L_FAST_PTE_SH_NS (((av8l_fast_iopte)0) << 8)
|
||||
#define AV8L_FAST_PTE_SH_OS (((av8l_fast_iopte)2) << 8)
|
||||
#define AV8L_FAST_PTE_SH_IS (((av8l_fast_iopte)3) << 8)
|
||||
#define AV8L_FAST_PTE_SH_MASK (((av8l_fast_iopte)3) << 8)
|
||||
#define AV8L_FAST_PTE_NS (((av8l_fast_iopte)1) << 5)
|
||||
#define AV8L_FAST_PTE_VALID (((av8l_fast_iopte)1) << 0)
|
||||
|
||||
#define AV8L_FAST_PTE_ATTR_LO_MASK (((av8l_fast_iopte)0x3ff) << 2)
|
||||
/* Ignore the contiguous bit for block splitting */
|
||||
#define AV8L_FAST_PTE_ATTR_HI_MASK (((av8l_fast_iopte)6) << 52)
|
||||
#define AV8L_FAST_PTE_ATTR_MASK (AV8L_FAST_PTE_ATTR_LO_MASK | \
|
||||
AV8L_FAST_PTE_ATTR_HI_MASK)
|
||||
#define AV8L_FAST_PTE_ADDR_MASK ((av8l_fast_iopte)0xfffffffff000)
|
||||
|
||||
|
||||
/* Stage-1 PTE */
|
||||
#define AV8L_FAST_PTE_AP_PRIV_RW (((av8l_fast_iopte)0) << 6)
|
||||
#define AV8L_FAST_PTE_AP_RW (((av8l_fast_iopte)1) << 6)
|
||||
#define AV8L_FAST_PTE_AP_PRIV_RO (((av8l_fast_iopte)2) << 6)
|
||||
#define AV8L_FAST_PTE_AP_RO (((av8l_fast_iopte)3) << 6)
|
||||
#define AV8L_FAST_PTE_ATTRINDX_SHIFT 2
|
||||
#define AV8L_FAST_PTE_ATTRINDX_MASK 0x7
|
||||
#define AV8L_FAST_PTE_nG (((av8l_fast_iopte)1) << 11)
|
||||
|
||||
/* Stage-2 PTE */
|
||||
#define AV8L_FAST_PTE_HAP_FAULT (((av8l_fast_iopte)0) << 6)
|
||||
#define AV8L_FAST_PTE_HAP_READ (((av8l_fast_iopte)1) << 6)
|
||||
#define AV8L_FAST_PTE_HAP_WRITE (((av8l_fast_iopte)2) << 6)
|
||||
#define AV8L_FAST_PTE_MEMATTR_OIWB (((av8l_fast_iopte)0xf) << 2)
|
||||
#define AV8L_FAST_PTE_MEMATTR_NC (((av8l_fast_iopte)0x5) << 2)
|
||||
#define AV8L_FAST_PTE_MEMATTR_DEV (((av8l_fast_iopte)0x1) << 2)
|
||||
|
||||
/* Register bits */
|
||||
#define ARM_32_LPAE_TCR_EAE (1 << 31)
|
||||
#define ARM_64_LPAE_S2_TCR_RES1 (1 << 31)
|
||||
|
||||
#define AV8L_FAST_TCR_TG0_4K (0 << 14)
|
||||
#define AV8L_FAST_TCR_TG0_64K (1 << 14)
|
||||
#define AV8L_FAST_TCR_TG0_16K (2 << 14)
|
||||
|
||||
#define AV8L_FAST_TCR_SH0_SHIFT 12
|
||||
#define AV8L_FAST_TCR_SH0_MASK 0x3
|
||||
#define AV8L_FAST_TCR_SH_NS 0
|
||||
#define AV8L_FAST_TCR_SH_OS 2
|
||||
#define AV8L_FAST_TCR_SH_IS 3
|
||||
|
||||
#define AV8L_FAST_TCR_ORGN0_SHIFT 10
|
||||
#define AV8L_FAST_TCR_IRGN0_SHIFT 8
|
||||
#define AV8L_FAST_TCR_RGN_MASK 0x3
|
||||
#define AV8L_FAST_TCR_RGN_NC 0
|
||||
#define AV8L_FAST_TCR_RGN_WBWA 1
|
||||
#define AV8L_FAST_TCR_RGN_WT 2
|
||||
#define AV8L_FAST_TCR_RGN_WB 3
|
||||
|
||||
#define AV8L_FAST_TCR_SL0_SHIFT 6
|
||||
#define AV8L_FAST_TCR_SL0_MASK 0x3
|
||||
|
||||
#define AV8L_FAST_TCR_T0SZ_SHIFT 0
|
||||
#define AV8L_FAST_TCR_SZ_MASK 0xf
|
||||
|
||||
#define AV8L_FAST_TCR_PS_SHIFT 16
|
||||
#define AV8L_FAST_TCR_PS_MASK 0x7
|
||||
|
||||
#define AV8L_FAST_TCR_IPS_SHIFT 32
|
||||
#define AV8L_FAST_TCR_IPS_MASK 0x7
|
||||
|
||||
#define AV8L_FAST_TCR_PS_32_BIT 0x0ULL
|
||||
#define AV8L_FAST_TCR_PS_36_BIT 0x1ULL
|
||||
#define AV8L_FAST_TCR_PS_40_BIT 0x2ULL
|
||||
#define AV8L_FAST_TCR_PS_42_BIT 0x3ULL
|
||||
#define AV8L_FAST_TCR_PS_44_BIT 0x4ULL
|
||||
#define AV8L_FAST_TCR_PS_48_BIT 0x5ULL
|
||||
|
||||
#define AV8L_FAST_TCR_EPD1_SHIFT 23
|
||||
#define AV8L_FAST_TCR_EPD1_FAULT 1
|
||||
|
||||
#define AV8L_FAST_MAIR_ATTR_SHIFT(n) ((n) << 3)
|
||||
#define AV8L_FAST_MAIR_ATTR_MASK 0xff
|
||||
#define AV8L_FAST_MAIR_ATTR_DEVICE 0x04
|
||||
#define AV8L_FAST_MAIR_ATTR_NC 0x44
|
||||
#define AV8L_FAST_MAIR_ATTR_WBRWA 0xff
|
||||
#define AV8L_FAST_MAIR_ATTR_UPSTREAM 0xf4
|
||||
#define AV8L_FAST_MAIR_ATTR_IDX_NC 0
|
||||
#define AV8L_FAST_MAIR_ATTR_IDX_CACHE 1
|
||||
#define AV8L_FAST_MAIR_ATTR_IDX_DEV 2
|
||||
#define AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM 3
|
||||
|
||||
#define AV8L_FAST_PAGE_SHIFT 12
|
||||
|
||||
#define PTE_MAIR_IDX(pte) \
|
||||
((pte >> AV8L_FAST_PTE_ATTRINDX_SHIFT) & \
|
||||
AV8L_FAST_PTE_ATTRINDX_MASK)
|
||||
|
||||
#define PTE_SH_IDX(pte) (pte & AV8L_FAST_PTE_SH_MASK)
|
||||
|
||||
#define iopte_pmd_offset(pmds, base, iova) (pmds + ((iova - base) >> 12))
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <linux/notifier.h>
|
||||
|
||||
static ATOMIC_NOTIFIER_HEAD(av8l_notifier_list);
|
||||
|
||||
void av8l_register_notify(struct notifier_block *nb)
|
||||
{
|
||||
atomic_notifier_chain_register(&av8l_notifier_list, nb);
|
||||
}
|
||||
EXPORT_SYMBOL(av8l_register_notify);
|
||||
|
||||
static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)
|
||||
{
|
||||
if (unlikely(*ptep)) {
|
||||
atomic_notifier_call_chain(
|
||||
&av8l_notifier_list, MAPPED_OVER_STALE_TLB,
|
||||
(void *) ptep);
|
||||
pr_err("Tried to map over a non-vacant pte: 0x%llx @ %p\n",
|
||||
*ptep, ptep);
|
||||
pr_err("Nearby memory:\n");
|
||||
print_hex_dump(KERN_ERR, "pgtbl: ", DUMP_PREFIX_ADDRESS,
|
||||
32, 8, ptep - 16, 32 * sizeof(*ptep), false);
|
||||
}
|
||||
}
|
||||
|
||||
void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base,
|
||||
u64 start, u64 end, bool skip_sync)
|
||||
{
|
||||
int i;
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
av8l_fast_iopte *pmdp = iopte_pmd_offset(pmds, base, start);
|
||||
|
||||
for (i = start >> AV8L_FAST_PAGE_SHIFT;
|
||||
i <= (end >> AV8L_FAST_PAGE_SHIFT); ++i) {
|
||||
if (!(*pmdp & AV8L_FAST_PTE_VALID)) {
|
||||
*pmdp = 0;
|
||||
if (!skip_sync)
|
||||
dmac_clean_range(pmdp, pmdp + 1);
|
||||
}
|
||||
pmdp++;
|
||||
}
|
||||
}
|
||||
#else
|
||||
static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
static void av8l_clean_range(struct io_pgtable_ops *ops,
|
||||
av8l_fast_iopte *start, av8l_fast_iopte *end)
|
||||
{
|
||||
struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops);
|
||||
|
||||
if (!(iop->cfg.quirks & IO_PGTABLE_QUIRK_NO_DMA))
|
||||
dmac_clean_range(start, end);
|
||||
}
|
||||
|
||||
static av8l_fast_iopte
|
||||
av8l_fast_prot_to_pte(struct av8l_fast_io_pgtable *data, int prot)
|
||||
{
|
||||
av8l_fast_iopte pte = AV8L_FAST_PTE_XN
|
||||
| AV8L_FAST_PTE_TYPE_PAGE
|
||||
| AV8L_FAST_PTE_AF
|
||||
| AV8L_FAST_PTE_nG
|
||||
| AV8L_FAST_PTE_SH_OS;
|
||||
|
||||
if (prot & IOMMU_MMIO)
|
||||
pte |= (AV8L_FAST_MAIR_ATTR_IDX_DEV
|
||||
<< AV8L_FAST_PTE_ATTRINDX_SHIFT);
|
||||
else if (prot & IOMMU_CACHE)
|
||||
pte |= (AV8L_FAST_MAIR_ATTR_IDX_CACHE
|
||||
<< AV8L_FAST_PTE_ATTRINDX_SHIFT);
|
||||
else if (prot & IOMMU_USE_UPSTREAM_HINT)
|
||||
pte |= (AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM
|
||||
<< AV8L_FAST_PTE_ATTRINDX_SHIFT);
|
||||
|
||||
if (!(prot & IOMMU_WRITE))
|
||||
pte |= AV8L_FAST_PTE_AP_RO;
|
||||
else
|
||||
pte |= AV8L_FAST_PTE_AP_RW;
|
||||
|
||||
return pte;
|
||||
}
|
||||
|
||||
static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);
|
||||
unsigned long i, nptes = size >> AV8L_FAST_PAGE_SHIFT;
|
||||
av8l_fast_iopte pte;
|
||||
|
||||
pte = av8l_fast_prot_to_pte(data, prot);
|
||||
paddr &= AV8L_FAST_PTE_ADDR_MASK;
|
||||
for (i = 0; i < nptes; i++, paddr += SZ_4K) {
|
||||
__av8l_check_for_stale_tlb(ptep + i);
|
||||
*(ptep + i) = pte | paddr;
|
||||
}
|
||||
av8l_clean_range(ops, ptep, ptep + nptes);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot)
|
||||
{
|
||||
return av8l_fast_map(ops, iova, paddr, size, prot);
|
||||
}
|
||||
|
||||
static size_t
|
||||
__av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
size_t size, bool allow_stale_tlb)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
unsigned long nptes;
|
||||
av8l_fast_iopte *ptep;
|
||||
int val = allow_stale_tlb
|
||||
? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI
|
||||
: 0;
|
||||
|
||||
ptep = iopte_pmd_offset(data->pmds, data->base, iova);
|
||||
nptes = size >> AV8L_FAST_PAGE_SHIFT;
|
||||
|
||||
memset(ptep, val, sizeof(*ptep) * nptes);
|
||||
av8l_clean_range(ops, ptep, ptep + nptes);
|
||||
if (!allow_stale_tlb)
|
||||
io_pgtable_tlb_flush_all(&data->iop);
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
/* caller must take care of tlb cache maintenance */
|
||||
void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
size_t size)
|
||||
{
|
||||
__av8l_fast_unmap(ops, iova, size, true);
|
||||
}
|
||||
|
||||
static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
size_t size)
|
||||
{
|
||||
return __av8l_fast_unmap(ops, iova, size, false);
|
||||
}
|
||||
|
||||
static int av8l_fast_map_sg(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, struct scatterlist *sgl,
|
||||
unsigned int nents, int prot, size_t *size)
|
||||
{
|
||||
struct scatterlist *sg;
|
||||
int i;
|
||||
|
||||
for_each_sg(sgl, sg, nents, i) {
|
||||
av8l_fast_map(ops, iova, sg_phys(sg), sg->length, prot);
|
||||
iova += sg->length;
|
||||
}
|
||||
|
||||
return nents;
|
||||
}
|
||||
|
||||
int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, struct scatterlist *sgl,
|
||||
unsigned int nents, int prot, size_t *size)
|
||||
{
|
||||
return av8l_fast_map_sg(ops, iova, sgl, nents, prot, size);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_ARM64)
|
||||
#define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27)
|
||||
#elif defined(CONFIG_ARM)
|
||||
#define FAST_PGDNDX(va) (((va) & 0xc0000000) >> 27)
|
||||
#endif
|
||||
|
||||
static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
av8l_fast_iopte pte, *pgdp, *pudp, *pmdp;
|
||||
unsigned long pgd;
|
||||
phys_addr_t phys;
|
||||
const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT;
|
||||
const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK;
|
||||
const unsigned long ptt = AV8L_FAST_PTE_TYPE_TABLE;
|
||||
const unsigned long ptp = AV8L_FAST_PTE_TYPE_PAGE;
|
||||
const av8l_fast_iopte am = AV8L_FAST_PTE_ADDR_MASK;
|
||||
|
||||
/* TODO: clean up some of these magic numbers... */
|
||||
|
||||
pgd = (unsigned long)data->pgd | FAST_PGDNDX(iova);
|
||||
pgdp = (av8l_fast_iopte *)pgd;
|
||||
|
||||
pte = *pgdp;
|
||||
if (((pte >> pts) & ptm) != ptt)
|
||||
return 0;
|
||||
pudp = phys_to_virt((pte & am) | ((iova & 0x3fe00000) >> 18));
|
||||
|
||||
pte = *pudp;
|
||||
if (((pte >> pts) & ptm) != ptt)
|
||||
return 0;
|
||||
pmdp = phys_to_virt((pte & am) | ((iova & 0x1ff000) >> 9));
|
||||
|
||||
pte = *pmdp;
|
||||
if (((pte >> pts) & ptm) != ptp)
|
||||
return 0;
|
||||
phys = pte & am;
|
||||
|
||||
return phys | (iova & 0xfff);
|
||||
}
|
||||
|
||||
phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
return av8l_fast_iova_to_phys(ops, iova);
|
||||
}
|
||||
|
||||
static bool av8l_fast_iova_coherent(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);
|
||||
|
||||
return ((PTE_MAIR_IDX(*ptep) == AV8L_FAST_MAIR_ATTR_IDX_CACHE) &&
|
||||
((PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_OS) ||
|
||||
(PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_IS)));
|
||||
}
|
||||
|
||||
bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
return av8l_fast_iova_coherent(ops, iova);
|
||||
}
|
||||
|
||||
static struct av8l_fast_io_pgtable *
|
||||
av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data;
|
||||
|
||||
data = kmalloc(sizeof(*data), GFP_KERNEL);
|
||||
if (!data)
|
||||
return NULL;
|
||||
|
||||
data->iop.ops = (struct io_pgtable_ops) {
|
||||
.map = av8l_fast_map,
|
||||
.map_sg = av8l_fast_map_sg,
|
||||
.unmap = av8l_fast_unmap,
|
||||
.iova_to_phys = av8l_fast_iova_to_phys,
|
||||
.is_iova_coherent = av8l_fast_iova_coherent,
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/*
|
||||
* We need max 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and
|
||||
* 2048 pages for pmds (each pud page contains 512 table entries, each
|
||||
* pointing to a pmd).
|
||||
*/
|
||||
#define NUM_PGD_PAGES 1
|
||||
#define NUM_PUD_PAGES 4
|
||||
#define NUM_PMD_PAGES 2048
|
||||
#define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES)
|
||||
|
||||
/* undefine arch specific definitions which depends on page table format */
|
||||
#undef pud_index
|
||||
#undef pud_mask
|
||||
#undef pud_next
|
||||
#undef pmd_index
|
||||
#undef pmd_mask
|
||||
#undef pmd_next
|
||||
|
||||
#define pud_index(addr) (((addr) >> 30) & 0x3)
|
||||
#define pud_mask(addr) ((addr) & ~((1UL << 30) - 1))
|
||||
#define pud_next(addr, end) \
|
||||
({ unsigned long __boundary = pud_mask(addr + (1UL << 30));\
|
||||
(__boundary - 1 < (end) - 1) ? __boundary : (end); \
|
||||
})
|
||||
|
||||
#define pmd_index(addr) (((addr) >> 21) & 0x1ff)
|
||||
#define pmd_mask(addr) ((addr) & ~((1UL << 21) - 1))
|
||||
#define pmd_next(addr, end) \
|
||||
({ unsigned long __boundary = pmd_mask(addr + (1UL << 21));\
|
||||
(__boundary - 1 < (end) - 1) ? __boundary : (end); \
|
||||
})
|
||||
|
||||
static int
|
||||
av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,
|
||||
struct io_pgtable_cfg *cfg, void *cookie)
|
||||
{
|
||||
int i, j, pg = 0;
|
||||
struct page **pages, *page;
|
||||
dma_addr_t base = cfg->iova_base;
|
||||
dma_addr_t end = cfg->iova_end;
|
||||
dma_addr_t pud, pmd;
|
||||
int pmd_pg_index;
|
||||
|
||||
pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, __GFP_NOWARN |
|
||||
__GFP_NORETRY);
|
||||
|
||||
if (!pages)
|
||||
pages = vmalloc(sizeof(*pages) * NUM_PGTBL_PAGES);
|
||||
|
||||
if (!pages)
|
||||
return -ENOMEM;
|
||||
|
||||
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
|
||||
if (!page)
|
||||
goto err_free_pages_arr;
|
||||
pages[pg++] = page;
|
||||
data->pgd = page_address(page);
|
||||
|
||||
/*
|
||||
* We need max 2048 entries at level 2 to map 4GB of VA space. A page
|
||||
* can hold 512 entries, so we need max 4 pages.
|
||||
*/
|
||||
for (i = pud_index(base), pud = base; pud < end;
|
||||
++i, pud = pud_next(pud, end)) {
|
||||
av8l_fast_iopte pte, *ptep;
|
||||
|
||||
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
|
||||
if (!page)
|
||||
goto err_free_pages;
|
||||
pages[pg++] = page;
|
||||
data->puds[i] = page_address(page);
|
||||
pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;
|
||||
ptep = ((av8l_fast_iopte *)data->pgd) + i;
|
||||
*ptep = pte;
|
||||
}
|
||||
dmac_clean_range(data->pgd, data->pgd + 4);
|
||||
|
||||
/*
|
||||
* We have max 4 puds, each of which can point to 512 pmds, so we'll
|
||||
* have max 2048 pmds, each of which can hold 512 ptes, for a grand
|
||||
* total of 2048*512=1048576 PTEs.
|
||||
*/
|
||||
pmd_pg_index = pg;
|
||||
for (i = pud_index(base), pud = base; pud < end;
|
||||
++i, pud = pud_next(pud, end)) {
|
||||
for (j = pmd_index(pud), pmd = pud; pmd < pud_next(pud, end);
|
||||
++j, pmd = pmd_next(pmd, end)) {
|
||||
av8l_fast_iopte pte, *pudp;
|
||||
void *addr;
|
||||
|
||||
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
|
||||
if (!page)
|
||||
goto err_free_pages;
|
||||
pages[pg++] = page;
|
||||
|
||||
addr = page_address(page);
|
||||
dmac_clean_range(addr, addr + SZ_4K);
|
||||
|
||||
pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;
|
||||
pudp = data->puds[i] + j;
|
||||
*pudp = pte;
|
||||
}
|
||||
dmac_clean_range(data->puds[i], data->puds[i] + 512);
|
||||
}
|
||||
|
||||
/*
|
||||
* We map the pmds into a virtually contiguous space so that we
|
||||
* don't have to traverse the first two levels of the page tables
|
||||
* to find the appropriate pud. Instead, it will be a simple
|
||||
* offset from the virtual base of the pmds.
|
||||
*/
|
||||
data->pmds = vmap(&pages[pmd_pg_index], pg - pmd_pg_index,
|
||||
VM_IOREMAP, PAGE_KERNEL);
|
||||
if (!data->pmds)
|
||||
goto err_free_pages;
|
||||
|
||||
data->pages = pages;
|
||||
data->nr_pages = pg;
|
||||
data->base = base;
|
||||
data->end = end;
|
||||
return 0;
|
||||
|
||||
err_free_pages:
|
||||
for (i = 0; i < pg; ++i)
|
||||
__free_page(pages[i]);
|
||||
err_free_pages_arr:
|
||||
kvfree(pages);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static struct io_pgtable *
|
||||
av8l_fast_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
|
||||
{
|
||||
u64 reg;
|
||||
struct av8l_fast_io_pgtable *data =
|
||||
av8l_fast_alloc_pgtable_data(cfg);
|
||||
|
||||
if (!data)
|
||||
return NULL;
|
||||
|
||||
/* restrict according to the fast map requirements */
|
||||
cfg->ias = 32;
|
||||
cfg->pgsize_bitmap = SZ_4K;
|
||||
|
||||
/* TCR */
|
||||
if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT)
|
||||
reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_ORGN0_SHIFT);
|
||||
else if (cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)
|
||||
reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_IRGN0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_ORGN0_SHIFT);
|
||||
else
|
||||
reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_ORGN0_SHIFT);
|
||||
|
||||
reg |= AV8L_FAST_TCR_TG0_4K;
|
||||
|
||||
switch (cfg->oas) {
|
||||
case 32:
|
||||
reg |= (AV8L_FAST_TCR_PS_32_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 36:
|
||||
reg |= (AV8L_FAST_TCR_PS_36_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 40:
|
||||
reg |= (AV8L_FAST_TCR_PS_40_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 42:
|
||||
reg |= (AV8L_FAST_TCR_PS_42_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 44:
|
||||
reg |= (AV8L_FAST_TCR_PS_44_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 48:
|
||||
reg |= (AV8L_FAST_TCR_PS_48_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
default:
|
||||
goto out_free_data;
|
||||
}
|
||||
|
||||
reg |= (64ULL - cfg->ias) << AV8L_FAST_TCR_T0SZ_SHIFT;
|
||||
reg |= AV8L_FAST_TCR_EPD1_FAULT << AV8L_FAST_TCR_EPD1_SHIFT;
|
||||
#if defined(CONFIG_ARM)
|
||||
reg |= ARM_32_LPAE_TCR_EAE;
|
||||
#endif
|
||||
cfg->av8l_fast_cfg.tcr = reg;
|
||||
|
||||
/* MAIRs */
|
||||
reg = (AV8L_FAST_MAIR_ATTR_NC
|
||||
<< AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_NC)) |
|
||||
(AV8L_FAST_MAIR_ATTR_WBRWA
|
||||
<< AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_CACHE)) |
|
||||
(AV8L_FAST_MAIR_ATTR_DEVICE
|
||||
<< AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_DEV)) |
|
||||
(AV8L_FAST_MAIR_ATTR_UPSTREAM
|
||||
<< AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM));
|
||||
|
||||
cfg->av8l_fast_cfg.mair[0] = reg;
|
||||
cfg->av8l_fast_cfg.mair[1] = 0;
|
||||
|
||||
/* Allocate all page table memory! */
|
||||
if (av8l_fast_prepopulate_pgtables(data, cfg, cookie))
|
||||
goto out_free_data;
|
||||
|
||||
cfg->av8l_fast_cfg.pmds = data->pmds;
|
||||
|
||||
/* TTBRs */
|
||||
cfg->av8l_fast_cfg.ttbr[0] = virt_to_phys(data->pgd);
|
||||
cfg->av8l_fast_cfg.ttbr[1] = 0;
|
||||
return &data->iop;
|
||||
|
||||
out_free_data:
|
||||
kfree(data);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void av8l_fast_free_pgtable(struct io_pgtable *iop)
|
||||
{
|
||||
int i;
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop);
|
||||
|
||||
vunmap(data->pmds);
|
||||
for (i = 0; i < data->nr_pages; ++i)
|
||||
__free_page(data->pages[i]);
|
||||
kvfree(data->pages);
|
||||
kfree(data);
|
||||
}
|
||||
|
||||
struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns = {
|
||||
.alloc = av8l_fast_alloc_pgtable,
|
||||
.free = av8l_fast_free_pgtable,
|
||||
};
|
||||
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_SELFTEST
|
||||
|
||||
#include <linux/dma-contiguous.h>
|
||||
|
||||
static struct io_pgtable_cfg *cfg_cookie;
|
||||
|
||||
static void dummy_tlb_flush_all(void *cookie)
|
||||
{
|
||||
WARN_ON(cookie != cfg_cookie);
|
||||
}
|
||||
|
||||
static void dummy_tlb_add_flush(unsigned long iova, size_t size, size_t granule,
|
||||
bool leaf, void *cookie)
|
||||
{
|
||||
WARN_ON(cookie != cfg_cookie);
|
||||
WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
|
||||
}
|
||||
|
||||
static void dummy_tlb_sync(void *cookie)
|
||||
{
|
||||
WARN_ON(cookie != cfg_cookie);
|
||||
}
|
||||
|
||||
static struct iommu_gather_ops dummy_tlb_ops __initdata = {
|
||||
.tlb_flush_all = dummy_tlb_flush_all,
|
||||
.tlb_add_flush = dummy_tlb_add_flush,
|
||||
.tlb_sync = dummy_tlb_sync,
|
||||
};
|
||||
|
||||
/*
|
||||
* Returns true if the iova range is successfully mapped to the contiguous
|
||||
* phys range in ops.
|
||||
*/
|
||||
static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops,
|
||||
const unsigned long iova_start,
|
||||
const phys_addr_t phys_start,
|
||||
const size_t size)
|
||||
{
|
||||
u64 iova = iova_start;
|
||||
phys_addr_t phys = phys_start;
|
||||
|
||||
while (iova < (iova_start + size)) {
|
||||
/* + 42 just to make sure offsetting is working */
|
||||
if (ops->iova_to_phys(ops, iova + 42) != (phys + 42))
|
||||
return false;
|
||||
iova += SZ_4K;
|
||||
phys += SZ_4K;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static int __init av8l_fast_positive_testing(void)
|
||||
{
|
||||
int failed = 0;
|
||||
u64 iova;
|
||||
struct io_pgtable_ops *ops;
|
||||
struct io_pgtable_cfg cfg;
|
||||
struct av8l_fast_io_pgtable *data;
|
||||
av8l_fast_iopte *pmds;
|
||||
u64 max = SZ_1G * 4ULL - 1;
|
||||
u64 base = 0;
|
||||
|
||||
cfg = (struct io_pgtable_cfg) {
|
||||
.quirks = 0,
|
||||
.tlb = &dummy_tlb_ops,
|
||||
.ias = 32,
|
||||
.oas = 32,
|
||||
.pgsize_bitmap = SZ_4K,
|
||||
.iova_base = base,
|
||||
.iova_end = max,
|
||||
};
|
||||
|
||||
cfg_cookie = &cfg;
|
||||
ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &cfg, &cfg);
|
||||
|
||||
if (WARN_ON(!ops))
|
||||
return 1;
|
||||
|
||||
data = iof_pgtable_ops_to_data(ops);
|
||||
pmds = data->pmds;
|
||||
|
||||
/* map the entire 4GB VA space with 4K map calls */
|
||||
for (iova = base; iova < max; iova += SZ_4K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
|
||||
base, max - base)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all */
|
||||
for (iova = base; iova < max; iova += SZ_4K) {
|
||||
if (WARN_ON(ops->unmap(ops, iova, SZ_4K) != SZ_4K))
|
||||
failed++;
|
||||
}
|
||||
|
||||
/* sweep up TLB proving PTEs */
|
||||
av8l_fast_clear_stale_ptes(pmds, base, base, max, false);
|
||||
|
||||
/* map the entire 4GB VA space with 8K map calls */
|
||||
for (iova = base; iova < max; iova += SZ_8K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
|
||||
base, max - base)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all with 8K unmap calls */
|
||||
for (iova = base; iova < max; iova += SZ_8K) {
|
||||
if (WARN_ON(ops->unmap(ops, iova, SZ_8K) != SZ_8K))
|
||||
failed++;
|
||||
}
|
||||
|
||||
/* sweep up TLB proving PTEs */
|
||||
av8l_fast_clear_stale_ptes(pmds, base, base, max, false);
|
||||
|
||||
/* map the entire 4GB VA space with 16K map calls */
|
||||
for (iova = base; iova < max; iova += SZ_16K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
|
||||
base, max - base)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all */
|
||||
for (iova = base; iova < max; iova += SZ_16K) {
|
||||
if (WARN_ON(ops->unmap(ops, iova, SZ_16K) != SZ_16K))
|
||||
failed++;
|
||||
}
|
||||
|
||||
/* sweep up TLB proving PTEs */
|
||||
av8l_fast_clear_stale_ptes(pmds, base, base, max, false);
|
||||
|
||||
/* map the entire 4GB VA space with 64K map calls */
|
||||
for (iova = base; iova < max; iova += SZ_64K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
|
||||
base, max - base)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all at once */
|
||||
if (WARN_ON(ops->unmap(ops, base, max - base) != (max - base)))
|
||||
failed++;
|
||||
|
||||
free_io_pgtable_ops(ops);
|
||||
return failed;
|
||||
}
|
||||
|
||||
static int __init av8l_fast_do_selftests(void)
|
||||
{
|
||||
int failed = 0;
|
||||
|
||||
failed += av8l_fast_positive_testing();
|
||||
|
||||
pr_err("selftest: completed with %d failures\n", failed);
|
||||
|
||||
return 0;
|
||||
}
|
||||
subsys_initcall(av8l_fast_do_selftests);
|
||||
#endif
|
||||
@@ -34,6 +34,9 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = {
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S
|
||||
[ARM_V7S] = &io_pgtable_arm_v7s_init_fns,
|
||||
#endif
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
|
||||
[ARM_V8L_FAST] = &io_pgtable_av8l_fast_init_fns,
|
||||
#endif
|
||||
};
|
||||
|
||||
struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
|
||||
|
||||
52
include/linux/dma-mapping-fast.h
Normal file
52
include/linux/dma-mapping-fast.h
Normal file
@@ -0,0 +1,52 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef __LINUX_DMA_MAPPING_FAST_H
|
||||
#define __LINUX_DMA_MAPPING_FAST_H
|
||||
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/io-pgtable-fast.h>
|
||||
|
||||
struct dma_iommu_mapping;
|
||||
struct io_pgtable_ops;
|
||||
struct iova_domain;
|
||||
|
||||
struct dma_fast_smmu_mapping {
|
||||
struct device *dev;
|
||||
struct iommu_domain *domain;
|
||||
struct iova_domain *iovad;
|
||||
|
||||
dma_addr_t base;
|
||||
size_t size;
|
||||
size_t num_4k_pages;
|
||||
|
||||
unsigned int bitmap_size;
|
||||
unsigned long *bitmap;
|
||||
unsigned long next_start;
|
||||
unsigned long upcoming_stale_bit;
|
||||
bool have_stale_tlbs;
|
||||
|
||||
dma_addr_t pgtbl_dma_handle;
|
||||
struct io_pgtable_ops *pgtbl_ops;
|
||||
|
||||
spinlock_t lock;
|
||||
struct notifier_block notifier;
|
||||
};
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
|
||||
int fast_smmu_init_mapping(struct device *dev,
|
||||
struct dma_iommu_mapping *mapping);
|
||||
void fast_smmu_put_dma_cookie(struct iommu_domain *domain);
|
||||
#else
|
||||
static inline int fast_smmu_init_mapping(struct device *dev,
|
||||
struct dma_iommu_mapping *mapping)
|
||||
{
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
static inline void fast_smmu_put_dma_cookie(struct iommu_domain *domain) {}
|
||||
#endif
|
||||
|
||||
#endif /* __LINUX_DMA_MAPPING_FAST_H */
|
||||
104
include/linux/io-pgtable-fast.h
Normal file
104
include/linux/io-pgtable-fast.h
Normal file
@@ -0,0 +1,104 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2017, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef __LINUX_IO_PGTABLE_FAST_H
|
||||
#define __LINUX_IO_PGTABLE_FAST_H
|
||||
|
||||
#include <linux/notifier.h>
|
||||
|
||||
/*
|
||||
* This ought to be private to io-pgtable-fast, but dma-mapping-fast
|
||||
* currently requires it for a debug usecase.
|
||||
*/
|
||||
typedef u64 av8l_fast_iopte;
|
||||
|
||||
struct io_pgtable_ops;
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
|
||||
|
||||
int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot);
|
||||
|
||||
void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
size_t size);
|
||||
|
||||
int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, struct scatterlist *sgl,
|
||||
unsigned int nents, int prot, size_t *size);
|
||||
|
||||
bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova);
|
||||
|
||||
phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova);
|
||||
#else
|
||||
static inline int
|
||||
av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
static inline void av8l_fast_unmap_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, size_t size)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, struct scatterlist *sgl,
|
||||
unsigned int nents, int prot, size_t *size)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
static inline phys_addr_t
|
||||
av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif /* CONFIG_IOMMU_IO_PGTABLE_FAST */
|
||||
|
||||
|
||||
/* events for notifiers passed to av8l_register_notify */
|
||||
#define MAPPED_OVER_STALE_TLB 1
|
||||
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB
|
||||
/*
|
||||
* Doesn't matter what we use as long as bit 0 is unset. The reason why we
|
||||
* need a different value at all is that there are certain hardware
|
||||
* platforms with erratum that require that a PTE actually be zero'd out
|
||||
* and not just have its valid bit unset.
|
||||
*/
|
||||
#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0xa
|
||||
|
||||
void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base,
|
||||
u64 start, u64 end, bool skip_sync);
|
||||
void av8l_register_notify(struct notifier_block *nb);
|
||||
|
||||
#else /* !CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */
|
||||
|
||||
#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0
|
||||
|
||||
static inline void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops,
|
||||
u64 base,
|
||||
u64 start,
|
||||
u64 end,
|
||||
bool skip_sync)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void av8l_register_notify(struct notifier_block *nb)
|
||||
{
|
||||
}
|
||||
|
||||
#endif /* CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */
|
||||
|
||||
#endif /* __LINUX_IO_PGTABLE_FAST_H */
|
||||
@@ -12,6 +12,7 @@ enum io_pgtable_fmt {
|
||||
ARM_64_LPAE_S1,
|
||||
ARM_64_LPAE_S2,
|
||||
ARM_V7S,
|
||||
ARM_V8L_FAST,
|
||||
IO_PGTABLE_NUM_FMTS,
|
||||
};
|
||||
|
||||
@@ -103,6 +104,13 @@ struct io_pgtable_cfg {
|
||||
u32 nmrr;
|
||||
u32 prrr;
|
||||
} arm_v7s_cfg;
|
||||
|
||||
struct {
|
||||
u64 ttbr[2];
|
||||
u64 tcr;
|
||||
u64 mair[2];
|
||||
void *pmds;
|
||||
} av8l_fast_cfg;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -204,5 +212,6 @@ extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
|
||||
extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
|
||||
extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
|
||||
extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
|
||||
extern struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns;
|
||||
|
||||
#endif /* __IO_PGTABLE_H */
|
||||
|
||||
Reference in New Issue
Block a user