iommu: elide flushing for higher order map/unmap operations

This patch removes any implicit flushing that occurs in the implementation
of map and unmap operations and adds new iommu_map/unmap() wrapper
functions. To maintain semantics of the iommu_legacy_map/unmap() wrapper
functions, these are modified to call the new wrapper functions and then
perform an explicit flush operation.

Because VT-d currently performs two different types of flush dependent upon
whether a PTE is being modified versus merely added (i.e. replacing a non-
present PTE) 'iommu flush flags' are defined by this patch and the
iommu_ops map_page() and unmap_page() methods are modified to OR the type
of flush necessary for the PTE that has been populated or depopulated into
an accumulated flags value. The accumulated value can then be passed into
the explicit flush operation.

The ARM SMMU implementations of map_page() and unmap_page() currently
perform no implicit flushing and therefore the modified methods do not
adjust the flush flags.

NOTE: The per-cpu 'iommu_dont_flush_iotlb' is respected by the
      iommu_legacy_map/unmap() wrapper functions and therefore this now
      applies to all IOMMU implementations rather than just VT-d.

Signed-off-by: Paul Durrant <paul.durrant@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Acked-by: Julien Grall <julien.grall@arm.com>
Acked-by: Brian Woods <brian.woods@amd.com>

diff --git a/xen/arch/arm/p2m.c b/xen/arch/arm/p2m.c
index 1e7c91e39a2aebd7bb78be2d2434ad0c3859f1f9..2394f974768148fe60a9ff6718db86b83cfb134b 100644 (file)
--- a/xen/arch/arm/p2m.c
+++ b/xen/arch/arm/p2m.c
@@ -1021,8 +1021,17 @@ static int __p2m_set_entry(struct p2m_domain *p2m,
 
     if ( need_iommu_pt_sync(p2m->domain) &&
          (lpae_is_valid(orig_pte) || lpae_is_valid(*entry)) )
+    {
+        unsigned int flush_flags = 0;
+
+        if ( lpae_is_valid(orig_pte) )
+            flush_flags |= IOMMU_FLUSHF_modified;
+        if ( lpae_is_valid(*entry) )
+            flush_flags |= IOMMU_FLUSHF_added;
+
         rc = iommu_iotlb_flush(p2m->domain, _dfn(gfn_x(sgfn)),
-                               1UL << page_order);
+                               1UL << page_order, flush_flags);
+    }
     else
         rc = 0;
 

diff --git a/xen/common/memory.c b/xen/common/memory.c
index f37eb288d4cf300c0cfe227cb3f045d006fdd12d..b6cf09585c4d8eec5221e2dfeea70900f71fec23 100644 (file)
--- a/xen/common/memory.c
+++ b/xen/common/memory.c
@@ -853,11 +853,13 @@ int xenmem_add_to_physmap(struct domain *d, struct xen_add_to_physmap *xatp,
 
         this_cpu(iommu_dont_flush_iotlb) = 0;
 
-        ret = iommu_flush(d, _dfn(xatp->idx - done), done);
+        ret = iommu_iotlb_flush(d, _dfn(xatp->idx - done), done,
+                                IOMMU_FLUSHF_added | IOMMU_FLUSHF_modified);
         if ( unlikely(ret) && rc >= 0 )
             rc = ret;
 
-        ret = iommu_flush(d, _dfn(xatp->gpfn - done), done);
+        ret = iommu_iotlb_flush(d, _dfn(xatp->gpfn - done), done,
+                                IOMMU_FLUSHF_added | IOMMU_FLUSHF_modified);
         if ( unlikely(ret) && rc >= 0 )
             rc = ret;
     }

diff --git a/xen/drivers/passthrough/amd/iommu_map.c b/xen/drivers/passthrough/amd/iommu_map.c
index de5a8800701be098edb403d69ae6ce6f7d81d9af..99ac0a68625ced2efd48a34e16bfc6938d0b7a80 100644 (file)
--- a/xen/drivers/passthrough/amd/iommu_map.c
+++ b/xen/drivers/passthrough/amd/iommu_map.c
@@ -35,23 +35,37 @@ static unsigned int pfn_to_pde_idx(unsigned long pfn, unsigned int level)
     return idx;
 }
 
-static void clear_iommu_pte_present(unsigned long l1_mfn, unsigned long dfn)
+static unsigned int clear_iommu_pte_present(unsigned long l1_mfn,
+                                            unsigned long dfn)
 {
     uint64_t *table, *pte;
+    uint32_t entry;
+    unsigned int flush_flags;
 
     table = map_domain_page(_mfn(l1_mfn));
-    pte = table + pfn_to_pde_idx(dfn, 1);
+
+    pte = (table + pfn_to_pde_idx(dfn, 1));
+    entry = *pte >> 32;
+
+    flush_flags = get_field_from_reg_u32(entry, IOMMU_PTE_PRESENT_MASK,
+                                         IOMMU_PTE_PRESENT_SHIFT) ?
+                                         IOMMU_FLUSHF_modified : 0;
+
     *pte = 0;
     unmap_domain_page(table);
+
+    return flush_flags;
 }
 
-static bool set_iommu_pde_present(uint32_t *pde, unsigned long next_mfn,
-                                  unsigned int next_level,
-                                  bool iw, bool ir)
+static unsigned int set_iommu_pde_present(uint32_t *pde,
+                                          unsigned long next_mfn,
+                                          unsigned int next_level, bool iw,
+                                          bool ir)
 {
     uint64_t maddr_next;
     uint32_t addr_lo, addr_hi, entry;
-    bool need_flush = false, old_present;
+    bool old_present;
+    unsigned int flush_flags = IOMMU_FLUSHF_added;
 
     maddr_next = __pfn_to_paddr(next_mfn);
 
@@ -84,7 +98,7 @@ static bool set_iommu_pde_present(uint32_t *pde, unsigned long next_mfn,
 
         if ( maddr_old != maddr_next || iw != old_w || ir != old_r ||
              old_level != next_level )
-            need_flush = true;
+            flush_flags |= IOMMU_FLUSHF_modified;
     }
 
     addr_lo = maddr_next & DMA_32BIT_MASK;
@@ -121,24 +135,27 @@ static bool set_iommu_pde_present(uint32_t *pde, unsigned long next_mfn,
                          IOMMU_PDE_PRESENT_SHIFT, &entry);
     pde[0] = entry;
 
-    return need_flush;
+    return flush_flags;
 }
 
-static bool set_iommu_pte_present(unsigned long pt_mfn, unsigned long dfn,
-                                  unsigned long next_mfn, int pde_level,
-                                  bool iw, bool ir)
+static unsigned int set_iommu_pte_present(unsigned long pt_mfn,
+                                          unsigned long dfn,
+                                          unsigned long next_mfn,
+                                          int pde_level,
+                                          bool iw, bool ir)
 {
     uint64_t *table;
     uint32_t *pde;
-    bool need_flush;
+    unsigned int flush_flags;
 
     table = map_domain_page(_mfn(pt_mfn));
 
     pde = (uint32_t *)(table + pfn_to_pde_idx(dfn, pde_level));
 
-    need_flush = set_iommu_pde_present(pde, next_mfn, 0, iw, ir);
+    flush_flags = set_iommu_pde_present(pde, next_mfn, 0, iw, ir);
     unmap_domain_page(table);
-    return need_flush;
+
+    return flush_flags;
 }
 
 void amd_iommu_set_root_page_table(uint32_t *dte, uint64_t root_ptr,
@@ -525,9 +542,8 @@ static int update_paging_mode(struct domain *d, unsigned long dfn)
 }
 
 int amd_iommu_map_page(struct domain *d, dfn_t dfn, mfn_t mfn,
-                       unsigned int flags)
+                       unsigned int flags, unsigned int *flush_flags)
 {
-    bool need_flush;
     struct domain_iommu *hd = dom_iommu(d);
     int rc;
     unsigned long pt_mfn[7];
@@ -573,18 +589,17 @@ int amd_iommu_map_page(struct domain *d, dfn_t dfn, mfn_t mfn,
     }
 
     /* Install 4k mapping */
-    need_flush = set_iommu_pte_present(pt_mfn[1], dfn_x(dfn), mfn_x(mfn), 1,
-                                       !!(flags & IOMMUF_writable),
-                                       !!(flags & IOMMUF_readable));
-
-    if ( need_flush )
-        amd_iommu_flush_pages(d, dfn_x(dfn), 0);
+    *flush_flags |= set_iommu_pte_present(pt_mfn[1], dfn_x(dfn), mfn_x(mfn),
+                                          1, (flags & IOMMUF_writable),
+                                          (flags & IOMMUF_readable));
 
     spin_unlock(&hd->arch.mapping_lock);
+
     return 0;
 }
 
-int amd_iommu_unmap_page(struct domain *d, dfn_t dfn)
+int amd_iommu_unmap_page(struct domain *d, dfn_t dfn,
+                         unsigned int *flush_flags)
 {
     unsigned long pt_mfn[7];
     struct domain_iommu *hd = dom_iommu(d);
@@ -629,11 +644,10 @@ int amd_iommu_unmap_page(struct domain *d, dfn_t dfn)
     }
 
     /* mark PTE as 'page not present' */
-    clear_iommu_pte_present(pt_mfn[1], dfn_x(dfn));
+    *flush_flags |= clear_iommu_pte_present(pt_mfn[1], dfn_x(dfn));
 
     spin_unlock(&hd->arch.mapping_lock);
 
-    amd_iommu_flush_pages(d, dfn_x(dfn), 0);
     return 0;
 }
 
@@ -648,11 +662,17 @@ static unsigned long flush_count(unsigned long dfn, unsigned int page_count,
 }
 
 int amd_iommu_flush_iotlb_pages(struct domain *d, dfn_t dfn,
-                                unsigned int page_count)
+                                unsigned int page_count,
+                                unsigned int flush_flags)
 {
     unsigned long dfn_l = dfn_x(dfn);
 
     ASSERT(page_count && !dfn_eq(dfn, INVALID_DFN));
+    ASSERT(flush_flags);
+
+    /* Unless a PTE was modified, no flush is required */
+    if ( !(flush_flags & IOMMU_FLUSHF_modified) )
+        return 0;
 
     /* If the range wraps then just flush everything */
     if ( dfn_l + page_count < dfn_l )
@@ -695,6 +715,7 @@ int amd_iommu_reserve_domain_unity_map(struct domain *domain,
     unsigned long npages, i;
     unsigned long gfn;
     unsigned int flags = !!ir;
+    unsigned int flush_flags = 0;
     int rt = 0;
 
     if ( iw )
@@ -706,11 +727,19 @@ int amd_iommu_reserve_domain_unity_map(struct domain *domain,
     {
         unsigned long frame = gfn + i;
 
-        rt = amd_iommu_map_page(domain, _dfn(frame), _mfn(frame), flags);
+        rt = amd_iommu_map_page(domain, _dfn(frame), _mfn(frame), flags,
+                                &flush_flags);
         if ( rt != 0 )
-            return rt;
+            break;
     }
-    return 0;
+
+    /* Use while-break to avoid compiler warning */
+    while ( flush_flags &&
+            amd_iommu_flush_iotlb_pages(domain, _dfn(gfn),
+                                        npages, flush_flags) )
+        break;
+
+    return rt;
 }
 
 /* Share p2m table with iommu. */

diff --git a/xen/drivers/passthrough/arm/smmu.c b/xen/drivers/passthrough/arm/smmu.c
index 9612c0fddcdcd8bfec7bcae2d5055ff32ab478ec..73c8048504b424d063983d3a08eab37ac2edd77e 100644 (file)
--- a/xen/drivers/passthrough/arm/smmu.c
+++ b/xen/drivers/passthrough/arm/smmu.c
@@ -2535,8 +2535,11 @@ static int __must_check arm_smmu_iotlb_flush_all(struct domain *d)
 }
 
 static int __must_check arm_smmu_iotlb_flush(struct domain *d, dfn_t dfn,
-                                             unsigned int page_count)
+                                            unsigned int page_count,
+                                            unsigned int flush_flags)
 {
+       ASSERT(flush_flags);
+
        /* ARM SMMU v1 doesn't have flush by VMA and VMID */
        return arm_smmu_iotlb_flush_all(d);
 }
@@ -2732,7 +2735,8 @@ static void arm_smmu_iommu_domain_teardown(struct domain *d)
 }
 
 static int __must_check arm_smmu_map_page(struct domain *d, dfn_t dfn,
-                                         mfn_t mfn, unsigned int flags)
+                                         mfn_t mfn, unsigned int flags,
+                                         unsigned int *flush_flags)
 {
        p2m_type_t t;
 
@@ -2761,7 +2765,8 @@ static int __must_check arm_smmu_map_page(struct domain *d, dfn_t dfn,
                                       0, t);
 }
 
-static int __must_check arm_smmu_unmap_page(struct domain *d, dfn_t dfn)
+static int __must_check arm_smmu_unmap_page(struct domain *d, dfn_t dfn,
+                                            unsigned int *flush_flags)
 {
        /*
         * This function should only be used by gnttab code when the domain

diff --git a/xen/drivers/passthrough/iommu.c b/xen/drivers/passthrough/iommu.c
index 105995a3438dc33ef73f811d8b8442dad74313f8..caff3ab24370f0ba029374d929368b3444bf4593 100644 (file)
--- a/xen/drivers/passthrough/iommu.c
+++ b/xen/drivers/passthrough/iommu.c
@@ -211,7 +211,7 @@ void __hwdom_init iommu_hwdom_init(struct domain *d)
     if ( need_iommu_pt_sync(d) )
     {
         struct page_info *page;
-        unsigned int i = 0;
+        unsigned int i = 0, flush_flags = 0;
         int rc = 0;
 
         page_list_for_each ( page, &d->page_list )
@@ -226,8 +226,9 @@ void __hwdom_init iommu_hwdom_init(struct domain *d)
                   == PGT_writable_page) )
                 mapping |= IOMMUF_writable;
 
-            ret = hd->platform_ops->map_page(d, _dfn(dfn), _mfn(mfn),
-                                             mapping);
+            ret = iommu_map(d, _dfn(dfn), _mfn(mfn), mapping, 0,
+                            &flush_flags);
+
             if ( !rc )
                 rc = ret;
 
@@ -235,6 +236,10 @@ void __hwdom_init iommu_hwdom_init(struct domain *d)
                 process_pending_softirqs();
         }
 
+        /* Use while-break to avoid compiler warning */
+        while ( iommu_iotlb_flush_all(d, flush_flags) )
+            break;
+
         if ( rc )
             printk(XENLOG_WARNING "d%d: IOMMU mapping failed: %d\n",
                    d->domain_id, rc);
@@ -304,8 +309,9 @@ void iommu_domain_destroy(struct domain *d)
     arch_iommu_domain_destroy(d);
 }
 
-int iommu_legacy_map(struct domain *d, dfn_t dfn, mfn_t mfn,
-                     unsigned int page_order, unsigned int flags)
+int iommu_map(struct domain *d, dfn_t dfn, mfn_t mfn,
+              unsigned int page_order, unsigned int flags,
+              unsigned int *flush_flags)
 {
     const struct domain_iommu *hd = dom_iommu(d);
     unsigned long i;
@@ -319,8 +325,8 @@ int iommu_legacy_map(struct domain *d, dfn_t dfn, mfn_t mfn,
 
     for ( i = 0; i < (1ul << page_order); i++ )
     {
-        rc = hd->platform_ops->map_page(d, dfn_add(dfn, i),
-                                        mfn_add(mfn, i), flags);
+        rc = hd->platform_ops->map_page(d, dfn_add(dfn, i), mfn_add(mfn, i),
+                                        flags, flush_flags);
 
         if ( likely(!rc) )
             continue;
@@ -333,7 +339,8 @@ int iommu_legacy_map(struct domain *d, dfn_t dfn, mfn_t mfn,
 
         while ( i-- )
             /* if statement to satisfy __must_check */
-            if ( hd->platform_ops->unmap_page(d, dfn_add(dfn, i)) )
+            if ( hd->platform_ops->unmap_page(d, dfn_add(dfn, i),
+                                              flush_flags) )
                 continue;
 
         if ( !is_hardware_domain(d) )
@@ -345,7 +352,26 @@ int iommu_legacy_map(struct domain *d, dfn_t dfn, mfn_t mfn,
     return rc;
 }
 
-int iommu_legacy_unmap(struct domain *d, dfn_t dfn, unsigned int page_order)
+int iommu_legacy_map(struct domain *d, dfn_t dfn, mfn_t mfn,
+                     unsigned int page_order, unsigned int flags)
+{
+    unsigned int flush_flags = 0;
+    int rc = iommu_map(d, dfn, mfn, page_order, flags, &flush_flags);
+
+    if ( !this_cpu(iommu_dont_flush_iotlb) )
+    {
+        int err = iommu_iotlb_flush(d, dfn, (1u << page_order),
+                                    flush_flags);
+
+        if ( !rc )
+            rc = err;
+    }
+
+    return rc;
+}
+
+int iommu_unmap(struct domain *d, dfn_t dfn, unsigned int page_order,
+                unsigned int *flush_flags)
 {
     const struct domain_iommu *hd = dom_iommu(d);
     unsigned long i;
@@ -358,7 +384,8 @@ int iommu_legacy_unmap(struct domain *d, dfn_t dfn, unsigned int page_order)
 
     for ( i = 0; i < (1ul << page_order); i++ )
     {
-        int err = hd->platform_ops->unmap_page(d, dfn_add(dfn, i));
+        int err = hd->platform_ops->unmap_page(d, dfn_add(dfn, i),
+                                               flush_flags);
 
         if ( likely(!err) )
             continue;
@@ -381,6 +408,23 @@ int iommu_legacy_unmap(struct domain *d, dfn_t dfn, unsigned int page_order)
     return rc;
 }
 
+int iommu_legacy_unmap(struct domain *d, dfn_t dfn, unsigned int page_order)
+{
+    unsigned int flush_flags = 0;
+    int rc = iommu_unmap(d, dfn, page_order, &flush_flags);
+
+    if ( !this_cpu(iommu_dont_flush_iotlb) )
+    {
+        int err = iommu_iotlb_flush(d, dfn, (1u << page_order),
+                                    flush_flags);
+
+        if ( !rc )
+            rc = err;
+    }
+
+    return rc;
+}
+
 int iommu_lookup_page(struct domain *d, dfn_t dfn, mfn_t *mfn,
                       unsigned int *flags)
 {
@@ -409,25 +453,26 @@ static void iommu_free_pagetables(unsigned long unused)
                             cpumask_cycle(smp_processor_id(), &cpu_online_map));
 }
 
-int iommu_iotlb_flush(struct domain *d, dfn_t dfn, unsigned int page_count)
+int iommu_iotlb_flush(struct domain *d, dfn_t dfn, unsigned int page_count,
+                      unsigned int flush_flags)
 {
     const struct domain_iommu *hd = dom_iommu(d);
     int rc;
 
     if ( !iommu_enabled || !hd->platform_ops ||
-         !hd->platform_ops->iotlb_flush || !page_count )
+         !hd->platform_ops->iotlb_flush || !page_count || !flush_flags )
         return 0;
 
     if ( dfn_eq(dfn, INVALID_DFN) )
         return -EINVAL;
 
-    rc = hd->platform_ops->iotlb_flush(d, dfn, page_count);
+    rc = hd->platform_ops->iotlb_flush(d, dfn, page_count, flush_flags);
     if ( unlikely(rc) )
     {
         if ( !d->is_shutting_down && printk_ratelimit() )
             printk(XENLOG_ERR
-                   "d%d: IOMMU IOTLB flush failed: %d, dfn %"PRI_dfn", page count %u\n",
-                   d->domain_id, rc, dfn_x(dfn), page_count);
+                   "d%d: IOMMU IOTLB flush failed: %d, dfn %"PRI_dfn", page count %u flags %x\n",
+                   d->domain_id, rc, dfn_x(dfn), page_count, flush_flags);
 
         if ( !is_hardware_domain(d) )
             domain_crash(d);
@@ -436,14 +481,19 @@ int iommu_iotlb_flush(struct domain *d, dfn_t dfn, unsigned int page_count)
     return rc;
 }
 
-int iommu_iotlb_flush_all(struct domain *d)
+int iommu_iotlb_flush_all(struct domain *d, unsigned int flush_flags)
 {
     const struct domain_iommu *hd = dom_iommu(d);
     int rc;
 
-    if ( !iommu_enabled || !hd->platform_ops || !hd->platform_ops->iotlb_flush_all )
+    if ( !iommu_enabled || !hd->platform_ops ||
+         !hd->platform_ops->iotlb_flush_all || !flush_flags )
         return 0;
 
+    /*
+     * The operation does a full flush so we don't need to pass the
+     * flush_flags in.
+     */
     rc = hd->platform_ops->iotlb_flush_all(d);
     if ( unlikely(rc) )
     {

diff --git a/xen/drivers/passthrough/vtd/iommu.c b/xen/drivers/passthrough/vtd/iommu.c
index d2fa5e2b25a7a35b88d4b237ff8b1e7df8b437fd..50a0e25224935aca903ba66bccba75b4d72f8fca 100644 (file)
--- a/xen/drivers/passthrough/vtd/iommu.c
+++ b/xen/drivers/passthrough/vtd/iommu.c
@@ -633,11 +633,14 @@ static int __must_check iommu_flush_iotlb(struct domain *d, dfn_t dfn,
 
 static int __must_check iommu_flush_iotlb_pages(struct domain *d,
                                                 dfn_t dfn,
-                                                unsigned int page_count)
+                                                unsigned int page_count,
+                                                unsigned int flush_flags)
 {
     ASSERT(page_count && !dfn_eq(dfn, INVALID_DFN));
+    ASSERT(flush_flags);
 
-    return iommu_flush_iotlb(d, dfn, 1, page_count);
+    return iommu_flush_iotlb(d, dfn, flush_flags & IOMMU_FLUSHF_modified,
+                             page_count);
 }
 
 static int __must_check iommu_flush_iotlb_all(struct domain *d)
@@ -646,7 +649,8 @@ static int __must_check iommu_flush_iotlb_all(struct domain *d)
 }
 
 /* clear one page's page table */
-static int __must_check dma_pte_clear_one(struct domain *domain, u64 addr)
+static int __must_check dma_pte_clear_one(struct domain *domain, u64 addr,
+                                          unsigned int *flush_flags)
 {
     struct domain_iommu *hd = dom_iommu(domain);
     struct dma_pte *page = NULL, *pte = NULL;
@@ -673,12 +677,11 @@ static int __must_check dma_pte_clear_one(struct domain *domain, u64 addr)
     }
 
     dma_clear_pte(*pte);
+    *flush_flags |= IOMMU_FLUSHF_modified;
+
     spin_unlock(&hd->arch.mapping_lock);
     iommu_flush_cache_entry(pte, sizeof(struct dma_pte));
 
-    if ( !this_cpu(iommu_dont_flush_iotlb) )
-        rc = iommu_flush_iotlb_pages(domain, daddr_to_dfn(addr), 1);
-
     unmap_vtd_domain_page(page);
 
     return rc;
@@ -1773,9 +1776,9 @@ static void iommu_domain_teardown(struct domain *d)
     spin_unlock(&hd->arch.mapping_lock);
 }
 
-static int __must_check intel_iommu_map_page(struct domain *d,
-                                             dfn_t dfn, mfn_t mfn,
-                                             unsigned int flags)
+static int __must_check intel_iommu_map_page(struct domain *d, dfn_t dfn,
+                                             mfn_t mfn, unsigned int flags,
+                                             unsigned int *flush_flags)
 {
     struct domain_iommu *hd = dom_iommu(d);
     struct dma_pte *page, *pte, old, new = {};
@@ -1825,14 +1828,15 @@ static int __must_check intel_iommu_map_page(struct domain *d,
     spin_unlock(&hd->arch.mapping_lock);
     unmap_vtd_domain_page(page);
 
-    if ( !this_cpu(iommu_dont_flush_iotlb) )
-        rc = iommu_flush_iotlb(d, dfn, dma_pte_present(old), 1);
+    *flush_flags |= IOMMU_FLUSHF_added;
+    if ( dma_pte_present(old) )
+        *flush_flags |= IOMMU_FLUSHF_modified;
 
     return rc;
 }
 
-static int __must_check intel_iommu_unmap_page(struct domain *d,
-                                               dfn_t dfn)
+static int __must_check intel_iommu_unmap_page(struct domain *d, dfn_t dfn,
+                                               unsigned int *flush_flags)
 {
     /* Do nothing if VT-d shares EPT page table */
     if ( iommu_use_hap_pt(d) )
@@ -1842,7 +1846,7 @@ static int __must_check intel_iommu_unmap_page(struct domain *d,
     if ( iommu_hwdom_passthrough && is_hardware_domain(d) )
         return 0;
 
-    return dma_pte_clear_one(d, dfn_to_daddr(dfn));
+    return dma_pte_clear_one(d, dfn_to_daddr(dfn), flush_flags);
 }
 
 static int intel_iommu_lookup_page(struct domain *d, dfn_t dfn, mfn_t *mfn,

diff --git a/xen/drivers/passthrough/x86/iommu.c b/xen/drivers/passthrough/x86/iommu.c
index b12289a18f7111e058ff70112b813043aeb06d12..e40d7a7d7befbe052f266fcc6b5e3ea3b3c034b2 100644 (file)
--- a/xen/drivers/passthrough/x86/iommu.c
+++ b/xen/drivers/passthrough/x86/iommu.c
@@ -46,11 +46,9 @@ int __init iommu_setup_hpet_msi(struct msi_desc *msi)
 
 int arch_iommu_populate_page_table(struct domain *d)
 {
-    const struct domain_iommu *hd = dom_iommu(d);
     struct page_info *page;
     int rc = 0, n = 0;
 
-    this_cpu(iommu_dont_flush_iotlb) = 1;
     spin_lock(&d->page_alloc_lock);
 
     if ( unlikely(d->is_dying) )
@@ -63,14 +61,15 @@ int arch_iommu_populate_page_table(struct domain *d)
         {
             unsigned long mfn = mfn_x(page_to_mfn(page));
             unsigned long gfn = mfn_to_gmfn(d, mfn);
+            unsigned int flush_flags = 0;
 
             if ( gfn != gfn_x(INVALID_GFN) )
             {
                 ASSERT(!(gfn >> DEFAULT_DOMAIN_ADDRESS_WIDTH));
                 BUG_ON(SHARED_M2P(gfn));
-                rc = hd->platform_ops->map_page(d, _dfn(gfn), _mfn(mfn),
-                                                IOMMUF_readable |
-                                                IOMMUF_writable);
+                rc = iommu_map(d, _dfn(gfn), _mfn(mfn), PAGE_ORDER_4K,
+                               IOMMUF_readable | IOMMUF_writable,
+                               &flush_flags);
             }
             if ( rc )
             {
@@ -104,10 +103,14 @@ int arch_iommu_populate_page_table(struct domain *d)
     }
 
     spin_unlock(&d->page_alloc_lock);
-    this_cpu(iommu_dont_flush_iotlb) = 0;
 
     if ( !rc )
-        rc = iommu_iotlb_flush_all(d);
+        /*
+         * flush_flags are not tracked across hypercall pre-emption so
+         * assume a full flush is necessary.
+         */
+        rc = iommu_iotlb_flush_all(
+            d, IOMMU_FLUSHF_added | IOMMU_FLUSHF_modified);
 
     if ( rc && rc != -ERESTART )
         iommu_teardown(d);
@@ -207,6 +210,7 @@ static bool __hwdom_init hwdom_iommu_map(const struct domain *d,
 void __hwdom_init arch_iommu_hwdom_init(struct domain *d)
 {
     unsigned long i, top, max_pfn;
+    unsigned int flush_flags = 0;
 
     BUG_ON(!is_hardware_domain(d));
 
@@ -241,8 +245,9 @@ void __hwdom_init arch_iommu_hwdom_init(struct domain *d)
         if ( paging_mode_translate(d) )
             rc = set_identity_p2m_entry(d, pfn, p2m_access_rw, 0);
         else
-            rc = iommu_legacy_map(d, _dfn(pfn), _mfn(pfn), PAGE_ORDER_4K,
-                                  IOMMUF_readable | IOMMUF_writable);
+            rc = iommu_map(d, _dfn(pfn), _mfn(pfn), PAGE_ORDER_4K,
+                           IOMMUF_readable | IOMMUF_writable, &flush_flags);
+
         if ( rc )
             printk(XENLOG_WARNING " d%d: IOMMU mapping failed: %d\n",
                    d->domain_id, rc);
@@ -250,6 +255,10 @@ void __hwdom_init arch_iommu_hwdom_init(struct domain *d)
         if (!(i & 0xfffff))
             process_pending_softirqs();
     }
+
+    /* Use if to avoid compiler warning */
+    if ( iommu_iotlb_flush_all(d, flush_flags) )
+        return;
 }
 
 /*

diff --git a/xen/include/asm-x86/hvm/svm/amd-iommu-proto.h b/xen/include/asm-x86/hvm/svm/amd-iommu-proto.h
index 88715329ca15e1e9dbd3d3a62772a3de33838feb..c5697565d65ce2c5cb464cca2156d6f50d1c34df 100644 (file)
--- a/xen/include/asm-x86/hvm/svm/amd-iommu-proto.h
+++ b/xen/include/asm-x86/hvm/svm/amd-iommu-proto.h
@@ -53,15 +53,18 @@ int amd_iommu_update_ivrs_mapping_acpi(void);
 
 /* mapping functions */
 int __must_check amd_iommu_map_page(struct domain *d, dfn_t dfn,
-                                    mfn_t mfn, unsigned int flags);
-int __must_check amd_iommu_unmap_page(struct domain *d, dfn_t dfn);
+                                    mfn_t mfn, unsigned int flags,
+                                    unsigned int *flush_flags);
+int __must_check amd_iommu_unmap_page(struct domain *d, dfn_t dfn,
+                                      unsigned int *flush_flags);
 uint64_t amd_iommu_get_address_from_pte(void *entry);
 int __must_check amd_iommu_alloc_root(struct domain_iommu *hd);
 int amd_iommu_reserve_domain_unity_map(struct domain *domain,
                                        paddr_t phys_addr, unsigned long size,
                                        int iw, int ir);
 int __must_check amd_iommu_flush_iotlb_pages(struct domain *d, dfn_t dfn,
-                                             unsigned int page_count);
+                                             unsigned int page_count,
+                                             unsigned int flush_flags);
 int __must_check amd_iommu_flush_iotlb_all(struct domain *d);
 
 /* Share p2m table with iommu */

diff --git a/xen/include/xen/iommu.h b/xen/include/xen/iommu.h
index 1f875aa328b9237031b543a15fa0b3a5528f4679..cdc8021cbdd64c39528cafb882761b1937b3d11a 100644 (file)
--- a/xen/include/xen/iommu.h
+++ b/xen/include/xen/iommu.h
@@ -92,6 +92,31 @@ void iommu_teardown(struct domain *d);
 #define _IOMMUF_writable 1
 #define IOMMUF_writable  (1u<<_IOMMUF_writable)
 
+/*
+ * flush_flags:
+ *
+ * IOMMU_FLUSHF_added -> A new 'present' PTE has been inserted.
+ * IOMMU_FLUSHF_modified -> An existing 'present' PTE has been modified
+ *                          (whether the new PTE value is 'present' or not).
+ *
+ * These flags are passed back from map/unmap operations and passed into
+ * flush operations.
+ */
+enum
+{
+    _IOMMU_FLUSHF_added,
+    _IOMMU_FLUSHF_modified,
+};
+#define IOMMU_FLUSHF_added (1u << _IOMMU_FLUSHF_added)
+#define IOMMU_FLUSHF_modified (1u << _IOMMU_FLUSHF_modified)
+
+int __must_check iommu_map(struct domain *d, dfn_t dfn, mfn_t mfn,
+                           unsigned int page_order, unsigned int flags,
+                           unsigned int *flush_flags);
+int __must_check iommu_unmap(struct domain *d, dfn_t dfn,
+                             unsigned int page_order,
+                             unsigned int *flush_flags);
+
 int __must_check iommu_legacy_map(struct domain *d, dfn_t dfn, mfn_t mfn,
                                   unsigned int page_order,
                                   unsigned int flags);
@@ -101,6 +126,12 @@ int __must_check iommu_legacy_unmap(struct domain *d, dfn_t dfn,
 int __must_check iommu_lookup_page(struct domain *d, dfn_t dfn, mfn_t *mfn,
                                    unsigned int *flags);
 
+int __must_check iommu_iotlb_flush(struct domain *d, dfn_t dfn,
+                                   unsigned int page_count,
+                                   unsigned int flush_flags);
+int __must_check iommu_iotlb_flush_all(struct domain *d,
+                                       unsigned int flush_flags);
+
 enum iommu_feature
 {
     IOMMU_FEAT_COHERENT_WALK,
@@ -178,8 +209,10 @@ struct iommu_ops {
      * other by the caller in order to have meaningful results.
      */
     int __must_check (*map_page)(struct domain *d, dfn_t dfn, mfn_t mfn,
-                                 unsigned int flags);
-    int __must_check (*unmap_page)(struct domain *d, dfn_t dfn);
+                                 unsigned int flags,
+                                 unsigned int *flush_flags);
+    int __must_check (*unmap_page)(struct domain *d, dfn_t dfn,
+                                   unsigned int *flush_flags);
     int __must_check (*lookup_page)(struct domain *d, dfn_t dfn, mfn_t *mfn,
                                     unsigned int *flags);
 
@@ -194,7 +227,8 @@ struct iommu_ops {
     void (*share_p2m)(struct domain *d);
     void (*crash_shutdown)(void);
     int __must_check (*iotlb_flush)(struct domain *d, dfn_t dfn,
-                                    unsigned int page_count);
+                                    unsigned int page_count,
+                                    unsigned int flush_flags);
     int __must_check (*iotlb_flush_all)(struct domain *d);
     int (*get_reserved_device_memory)(iommu_grdm_t *, void *);
     void (*dump_p2m_table)(struct domain *d);
@@ -253,10 +287,6 @@ int iommu_do_pci_domctl(struct xen_domctl *, struct domain *d,
 int iommu_do_domctl(struct xen_domctl *, struct domain *d,
                     XEN_GUEST_HANDLE_PARAM(xen_domctl_t));
 
-int __must_check iommu_iotlb_flush(struct domain *d, dfn_t dfn,
-                                   unsigned int page_count);
-int __must_check iommu_iotlb_flush_all(struct domain *d);
-
 void iommu_dev_iotlb_flush_timeout(struct domain *d, struct pci_dev *pdev);
 
 /*