From 2322545b20b3ae8bfd87c910487ac6afb7a7ea5e Mon Sep 17 00:00:00 2001 From: patrick Date: Fri, 25 Jun 2021 12:40:29 +0000 Subject: [PATCH] Save quite a bit of space by removing the existence of PTEDs. The dynamics of SMMU are a bit different to regular MMU usage, as we do not need P->V lists or ref/mod emulation (with page access upgrade). While in the future we might want to save cacheability modes, it is not necessary right now. Our PTED construct, which holds that kind of information, is not needed. With these gone, we save around 93% of smmu(4)'s previous memory overhead. Discussed with drahn@ kettenis@ --- sys/arch/arm64/dev/smmu.c | 193 ++++++++++++++--------------------- sys/arch/arm64/dev/smmuvar.h | 4 +- 2 files changed, 76 insertions(+), 121 deletions(-) diff --git a/sys/arch/arm64/dev/smmu.c b/sys/arch/arm64/dev/smmu.c index 9cf65a68a90..dd942391587 100644 --- a/sys/arch/arm64/dev/smmu.c +++ b/sys/arch/arm64/dev/smmu.c @@ -1,4 +1,4 @@ -/* $OpenBSD: smmu.c,v 1.15 2021/06/23 19:46:13 patrick Exp $ */ +/* $OpenBSD: smmu.c,v 1.16 2021/06/25 12:40:29 patrick Exp $ */ /* * Copyright (c) 2008-2009,2014-2016 Dale Rahn * Copyright (c) 2021 Patrick Wildt @@ -56,17 +56,11 @@ struct smmuvp2 { struct smmuvp3 { uint64_t l3[VP_IDX3_CNT]; - struct pte_desc *vp[VP_IDX3_CNT]; }; CTASSERT(sizeof(struct smmuvp0) == sizeof(struct smmuvp1)); CTASSERT(sizeof(struct smmuvp0) == sizeof(struct smmuvp2)); -CTASSERT(sizeof(struct smmuvp0) == sizeof(struct smmuvp3)); - -struct pte_desc { - uint64_t pted_pte; - vaddr_t pted_va; -}; +CTASSERT(sizeof(struct smmuvp0) != sizeof(struct smmuvp3)); uint32_t smmu_gr0_read_4(struct smmu_softc *, bus_size_t); void smmu_gr0_write_4(struct smmu_softc *, bus_size_t, uint32_t); @@ -89,14 +83,13 @@ void smmu_set_l2(struct smmu_domain *, uint64_t, struct smmuvp1 *, void smmu_set_l3(struct smmu_domain *, uint64_t, struct smmuvp2 *, struct smmuvp3 *); -struct pte_desc *smmu_vp_lookup(struct smmu_domain *, vaddr_t, uint64_t **); -int smmu_vp_enter(struct smmu_domain *, vaddr_t, struct pte_desc *, int); +int smmu_vp_lookup(struct smmu_domain *, vaddr_t, uint64_t **); +int smmu_vp_enter(struct smmu_domain *, vaddr_t, uint64_t **, int); -void smmu_fill_pte(struct smmu_domain *, vaddr_t, paddr_t, struct pte_desc *, +uint64_t smmu_fill_pte(struct smmu_domain *, vaddr_t, paddr_t, vm_prot_t, int, int); -void smmu_pte_update(struct smmu_domain *, struct pte_desc *, uint64_t *); -void smmu_pte_insert(struct smmu_domain *, struct pte_desc *); -void smmu_pte_remove(struct smmu_domain *, struct pte_desc *, int); +void smmu_pte_update(struct smmu_domain *, uint64_t, uint64_t *); +void smmu_pte_remove(struct smmu_domain *, vaddr_t); int smmu_enter(struct smmu_domain *, vaddr_t, paddr_t, vm_prot_t, int, int); void smmu_map(struct smmu_domain *, vaddr_t, paddr_t, vm_prot_t, int, int); @@ -133,12 +126,12 @@ smmu_attach(struct smmu_softc *sc) SIMPLEQ_INIT(&sc->sc_domains); - pool_init(&sc->sc_pted_pool, sizeof(struct pte_desc), 0, IPL_VM, 0, - "smmu_pted", NULL); - pool_setlowat(&sc->sc_pted_pool, 20); pool_init(&sc->sc_vp_pool, sizeof(struct smmuvp0), PAGE_SIZE, IPL_VM, 0, "smmu_vp", NULL); pool_setlowat(&sc->sc_vp_pool, 20); + pool_init(&sc->sc_vp3_pool, sizeof(struct smmuvp3), PAGE_SIZE, IPL_VM, 0, + "smmu_vp3", NULL); + pool_setlowat(&sc->sc_vp3_pool, 20); reg = smmu_gr0_read_4(sc, SMMU_IDR0); if (reg & SMMU_IDR0_S1TS) @@ -817,10 +810,10 @@ smmu_set_l1(struct smmu_domain *dom, uint64_t va, struct smmuvp1 *l1_va) int idx0; if (pmap_extract(pmap_kernel(), (vaddr_t)l1_va, &l1_pa) == 0) - panic("unable to find vp pa mapping %p", l1_va); + panic("%s: unable to find vp pa mapping %p", __func__, l1_va); if (l1_pa & (Lx_TABLE_ALIGN-1)) - panic("misaligned L2 table"); + panic("%s: misaligned L2 table", __func__); pg_entry = VP_Lx(l1_pa); @@ -838,10 +831,10 @@ smmu_set_l2(struct smmu_domain *dom, uint64_t va, struct smmuvp1 *vp1, int idx1; if (pmap_extract(pmap_kernel(), (vaddr_t)l2_va, &l2_pa) == 0) - panic("unable to find vp pa mapping %p", l2_va); + panic("%s: unable to find vp pa mapping %p", __func__, l2_va); if (l2_pa & (Lx_TABLE_ALIGN-1)) - panic("misaligned L2 table"); + panic("%s: misaligned L2 table", __func__); pg_entry = VP_Lx(l2_pa); @@ -859,10 +852,10 @@ smmu_set_l3(struct smmu_domain *dom, uint64_t va, struct smmuvp2 *vp2, int idx2; if (pmap_extract(pmap_kernel(), (vaddr_t)l3_va, &l3_pa) == 0) - panic("unable to find vp pa mapping %p", l3_va); + panic("%s: unable to find vp pa mapping %p", __func__, l3_va); if (l3_pa & (Lx_TABLE_ALIGN-1)) - panic("misaligned L2 table"); + panic("%s: misaligned L2 table", __func__); pg_entry = VP_Lx(l3_pa); @@ -871,45 +864,43 @@ smmu_set_l3(struct smmu_domain *dom, uint64_t va, struct smmuvp2 *vp2, vp2->l2[idx2] = pg_entry; } -struct pte_desc * +int smmu_vp_lookup(struct smmu_domain *dom, vaddr_t va, uint64_t **pl3entry) { struct smmuvp1 *vp1; struct smmuvp2 *vp2; struct smmuvp3 *vp3; - struct pte_desc *pted; if (dom->sd_4level) { if (dom->sd_vp.l0 == NULL) { - return NULL; + return ENXIO; } vp1 = dom->sd_vp.l0->vp[VP_IDX0(va)]; } else { vp1 = dom->sd_vp.l1; } if (vp1 == NULL) { - return NULL; + return ENXIO; } vp2 = vp1->vp[VP_IDX1(va)]; if (vp2 == NULL) { - return NULL; + return ENXIO; } vp3 = vp2->vp[VP_IDX2(va)]; if (vp3 == NULL) { - return NULL; + return ENXIO; } - pted = vp3->vp[VP_IDX3(va)]; if (pl3entry != NULL) *pl3entry = &(vp3->l3[VP_IDX3(va)]); - return pted; + return 0; } int -smmu_vp_enter(struct smmu_domain *dom, vaddr_t va, struct pte_desc *pted, +smmu_vp_enter(struct smmu_domain *dom, vaddr_t va, uint64_t **pl3entry, int flags) { struct smmu_softc *sc = dom->sd_sc; @@ -957,7 +948,7 @@ smmu_vp_enter(struct smmu_domain *dom, vaddr_t va, struct pte_desc *pted, mtx_enter(&dom->sd_pmap_mtx); vp3 = vp2->vp[VP_IDX2(va)]; if (vp3 == NULL) { - vp3 = pool_get(&sc->sc_vp_pool, PR_NOWAIT | PR_ZERO); + vp3 = pool_get(&sc->sc_vp3_pool, PR_NOWAIT | PR_ZERO); if (vp3 == NULL) { mtx_leave(&dom->sd_pmap_mtx); return ENOMEM; @@ -967,15 +958,19 @@ smmu_vp_enter(struct smmu_domain *dom, vaddr_t va, struct pte_desc *pted, mtx_leave(&dom->sd_pmap_mtx); } - vp3->vp[VP_IDX3(va)] = pted; + if (pl3entry != NULL) + *pl3entry = &(vp3->l3[VP_IDX3(va)]); + return 0; } -void +uint64_t smmu_fill_pte(struct smmu_domain *dom, vaddr_t va, paddr_t pa, - struct pte_desc *pted, vm_prot_t prot, int flags, int cache) + vm_prot_t prot, int flags, int cache) { - pted->pted_va = va; + uint64_t pted; + + pted = pa & PTE_RPGN; switch (cache) { case PMAP_CACHE_WB: @@ -991,22 +986,20 @@ smmu_fill_pte(struct smmu_domain *dom, vaddr_t va, paddr_t pa, default: panic("%s: invalid cache mode", __func__); } - pted->pted_va |= cache; - pted->pted_va |= prot & (PROT_READ|PROT_WRITE|PROT_EXEC); - - pted->pted_pte = pa & PTE_RPGN; - pted->pted_pte |= flags & (PROT_READ|PROT_WRITE|PROT_EXEC); + pted |= cache; + pted |= flags & (PROT_READ|PROT_WRITE|PROT_EXEC); + return pted; } void -smmu_pte_update(struct smmu_domain *dom, struct pte_desc *pted, uint64_t *pl3) +smmu_pte_update(struct smmu_domain *dom, uint64_t pted, uint64_t *pl3) { uint64_t pte, access_bits; uint64_t attr = 0; /* see mair in locore.S */ - switch (pted->pted_va & PMAP_CACHE_BITS) { + switch (pted & PMAP_CACHE_BITS) { case PMAP_CACHE_WB: /* inner and outer writeback */ if (dom->sd_stage == 1) @@ -1052,36 +1045,22 @@ smmu_pte_update(struct smmu_domain *dom, struct pte_desc *pted, uint64_t *pl3) if (dom->sd_stage == 1) { attr |= ATTR_nG; access_bits |= ATTR_AP(1); - if ((pted->pted_pte & PROT_READ) && - !(pted->pted_pte & PROT_WRITE)) + if ((pted & PROT_READ) && + !(pted & PROT_WRITE)) access_bits |= ATTR_AP(2); } else { - if (pted->pted_pte & PROT_READ) + if (pted & PROT_READ) access_bits |= ATTR_AP(1); - if (pted->pted_pte & PROT_WRITE) + if (pted & PROT_WRITE) access_bits |= ATTR_AP(2); } - pte = (pted->pted_pte & PTE_RPGN) | attr | access_bits | L3_P; + pte = (pted & PTE_RPGN) | attr | access_bits | L3_P; *pl3 = pte; } void -smmu_pte_insert(struct smmu_domain *dom, struct pte_desc *pted) -{ - uint64_t *pl3; - - if (smmu_vp_lookup(dom, pted->pted_va, &pl3) == NULL) { - panic("%s: have a pted, but missing a vp" - " for %lx va domain %p", __func__, pted->pted_va, dom); - } - - smmu_pte_update(dom, pted, pl3); - membar_producer(); /* XXX bus dma sync? */ -} - -void -smmu_pte_remove(struct smmu_domain *dom, struct pte_desc *pted, int remove_pted) +smmu_pte_remove(struct smmu_domain *dom, vaddr_t va) { /* put entry into table */ /* need to deal with ref/change here */ @@ -1090,92 +1069,81 @@ smmu_pte_remove(struct smmu_domain *dom, struct pte_desc *pted, int remove_pted) struct smmuvp3 *vp3; if (dom->sd_4level) - vp1 = dom->sd_vp.l0->vp[VP_IDX0(pted->pted_va)]; + vp1 = dom->sd_vp.l0->vp[VP_IDX0(va)]; else vp1 = dom->sd_vp.l1; if (vp1 == NULL) { - panic("have a pted, but missing the l1 for %lx va domain %p", - pted->pted_va, dom); + panic("%s: missing the l1 for va %lx domain %p", __func__, + va, dom); } - vp2 = vp1->vp[VP_IDX1(pted->pted_va)]; + vp2 = vp1->vp[VP_IDX1(va)]; if (vp2 == NULL) { - panic("have a pted, but missing the l2 for %lx va domain %p", - pted->pted_va, dom); + panic("%s: missing the l2 for va %lx domain %p", __func__, + va, dom); } - vp3 = vp2->vp[VP_IDX2(pted->pted_va)]; + vp3 = vp2->vp[VP_IDX2(va)]; if (vp3 == NULL) { - panic("have a pted, but missing the l3 for %lx va domain %p", - pted->pted_va, dom); + panic("%s: missing the l3 for va %lx domain %p", __func__, + va, dom); } - vp3->l3[VP_IDX3(pted->pted_va)] = 0; - if (remove_pted) - vp3->vp[VP_IDX3(pted->pted_va)] = NULL; + vp3->l3[VP_IDX3(va)] = 0; } int smmu_enter(struct smmu_domain *dom, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags, int cache) { - struct smmu_softc *sc = dom->sd_sc; - struct pte_desc *pted; - int error; + uint64_t *pl3; /* printf("%s: 0x%lx -> 0x%lx\n", __func__, va, pa); */ - pted = smmu_vp_lookup(dom, va, NULL); - if (pted == NULL) { - pted = pool_get(&sc->sc_pted_pool, PR_NOWAIT | PR_ZERO); - if (pted == NULL) { - error = ENOMEM; - goto out; - } - if (smmu_vp_enter(dom, va, pted, flags)) { - pool_put(&sc->sc_pted_pool, pted); - error = ENOMEM; - goto out; - } + if (smmu_vp_lookup(dom, va, &pl3) != 0) { + if (smmu_vp_enter(dom, va, &pl3, flags)) + return ENOMEM; } - smmu_fill_pte(dom, va, pa, pted, prot, flags, cache); + if (flags & (PROT_READ|PROT_WRITE|PROT_EXEC)) + smmu_map(dom, va, pa, prot, flags, cache); - error = 0; -out: - return error; + return 0; } void smmu_map(struct smmu_domain *dom, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags, int cache) { - struct pte_desc *pted; + uint64_t *pl3; + uint64_t pted; + int ret; /* printf("%s: 0x%lx -> 0x%lx\n", __func__, va, pa); */ /* IOVA must already be allocated */ - pted = smmu_vp_lookup(dom, va, NULL); - KASSERT(pted != NULL); + ret = smmu_vp_lookup(dom, va, &pl3); + KASSERT(ret == 0); /* Update PTED information for physical address */ - smmu_fill_pte(dom, va, pa, pted, prot, flags, cache); + pted = smmu_fill_pte(dom, va, pa, prot, flags, cache); /* Insert updated information */ - smmu_pte_insert(dom, pted); + smmu_pte_update(dom, pted, pl3); + membar_producer(); /* XXX bus dma sync? */ } void smmu_unmap(struct smmu_domain *dom, vaddr_t va) { struct smmu_softc *sc = dom->sd_sc; - struct pte_desc *pted; + int ret; /* printf("%s: 0x%lx\n", __func__, va); */ /* IOVA must already be allocated */ - pted = smmu_vp_lookup(dom, va, NULL); - KASSERT(pted != NULL); + ret = smmu_vp_lookup(dom, va, NULL); + KASSERT(ret == 0); - /* Remove mapping from pagetable, keep it alive */ - smmu_pte_remove(dom, pted, 0); + /* Remove mapping from pagetable */ + smmu_pte_remove(dom, va); membar_producer(); /* XXX bus dma sync? */ /* Invalidate IOTLB */ @@ -1190,22 +1158,9 @@ smmu_unmap(struct smmu_domain *dom, vaddr_t va) void smmu_remove(struct smmu_domain *dom, vaddr_t va) { - struct smmu_softc *sc = dom->sd_sc; - struct pte_desc *pted; - /* printf("%s: 0x%lx\n", __func__, va); */ - /* IOVA must already be allocated */ - pted = smmu_vp_lookup(dom, va, NULL); - KASSERT(pted != NULL); - - /* Mapping already removed, remove pted as well */ - smmu_pte_remove(dom, pted, 1); - - /* Destroy pted */ - pted->pted_pte = 0; - pted->pted_va = 0; - pool_put(&sc->sc_pted_pool, pted); + /* TODO: garbage collect page tables? */ } int diff --git a/sys/arch/arm64/dev/smmuvar.h b/sys/arch/arm64/dev/smmuvar.h index ca3a7b36d14..14dc0f20918 100644 --- a/sys/arch/arm64/dev/smmuvar.h +++ b/sys/arch/arm64/dev/smmuvar.h @@ -1,4 +1,4 @@ -/* $OpenBSD: smmuvar.h,v 1.4 2021/03/15 22:48:57 patrick Exp $ */ +/* $OpenBSD: smmuvar.h,v 1.5 2021/06/25 12:40:29 patrick Exp $ */ /* * Copyright (c) 2021 Patrick Wildt * @@ -71,8 +71,8 @@ struct smmu_softc { struct smmu_smr **sc_smr; struct smmu_cb **sc_cb; int sc_coherent; - struct pool sc_pted_pool; struct pool sc_vp_pool; + struct pool sc_vp3_pool; SIMPLEQ_HEAD(, smmu_domain) sc_domains; }; -- 2.20.1