IOMMU: patch certain indirect calls to direct ones

This is intentionally not touching hooks used rarely (or not at all)
during the lifetime of a VM, unless perhaps sitting on an error path
next to a call which gets changed (in which case I think the error
path better remains consistent with the respective main path).

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Wei Liu <wei.liu2@citrix.com>

diff --git a/xen/drivers/passthrough/iommu.c b/xen/drivers/passthrough/iommu.c
index 0d2dacf2879c29e231cb8d5f68489ca8619b19bc..79ec6719f5722179f1258b8385c207b284429b70 100644 (file)
--- a/xen/drivers/passthrough/iommu.c
+++ b/xen/drivers/passthrough/iommu.c
@@ -304,8 +304,8 @@ int iommu_map(struct domain *d, dfn_t dfn, mfn_t mfn,
 
     for ( i = 0; i < (1ul << page_order); i++ )
     {
-        rc = hd->platform_ops->map_page(d, dfn_add(dfn, i), mfn_add(mfn, i),
-                                        flags, flush_flags);
+        rc = iommu_call(hd->platform_ops, map_page, d, dfn_add(dfn, i),
+                        mfn_add(mfn, i), flags, flush_flags);
 
         if ( likely(!rc) )
             continue;
@@ -318,8 +318,8 @@ int iommu_map(struct domain *d, dfn_t dfn, mfn_t mfn,
 
         while ( i-- )
             /* if statement to satisfy __must_check */
-            if ( hd->platform_ops->unmap_page(d, dfn_add(dfn, i),
-                                              flush_flags) )
+            if ( iommu_call(hd->platform_ops, unmap_page, d, dfn_add(dfn, i),
+                            flush_flags) )
                 continue;
 
         if ( !is_hardware_domain(d) )
@@ -363,8 +363,8 @@ int iommu_unmap(struct domain *d, dfn_t dfn, unsigned int page_order,
 
     for ( i = 0; i < (1ul << page_order); i++ )
     {
-        int err = hd->platform_ops->unmap_page(d, dfn_add(dfn, i),
-                                               flush_flags);
+        int err = iommu_call(hd->platform_ops, unmap_page, d, dfn_add(dfn, i),
+                             flush_flags);
 
         if ( likely(!err) )
             continue;
@@ -412,7 +412,7 @@ int iommu_lookup_page(struct domain *d, dfn_t dfn, mfn_t *mfn,
     if ( !iommu_enabled || !hd->platform_ops || !hd->platform_ops->lookup_page )
         return -EOPNOTSUPP;
 
-    return hd->platform_ops->lookup_page(d, dfn, mfn, flags);
+    return iommu_call(hd->platform_ops, lookup_page, d, dfn, mfn, flags);
 }
 
 static void iommu_free_pagetables(unsigned long unused)
@@ -425,7 +425,7 @@ static void iommu_free_pagetables(unsigned long unused)
         spin_unlock(&iommu_pt_cleanup_lock);
         if ( !pg )
             return;
-        iommu_get_ops()->free_page_table(pg);
+        iommu_vcall(iommu_get_ops(), free_page_table, pg);
     } while ( !softirq_pending(smp_processor_id()) );
 
     tasklet_schedule_on_cpu(&iommu_pt_cleanup_tasklet,
@@ -445,7 +445,8 @@ int iommu_iotlb_flush(struct domain *d, dfn_t dfn, unsigned int page_count,
     if ( dfn_eq(dfn, INVALID_DFN) )
         return -EINVAL;
 
-    rc = hd->platform_ops->iotlb_flush(d, dfn, page_count, flush_flags);
+    rc = iommu_call(hd->platform_ops, iotlb_flush, d, dfn, page_count,
+                    flush_flags);
     if ( unlikely(rc) )
     {
         if ( !d->is_shutting_down && printk_ratelimit() )
@@ -473,7 +474,7 @@ int iommu_iotlb_flush_all(struct domain *d, unsigned int flush_flags)
      * The operation does a full flush so we don't need to pass the
      * flush_flags in.
      */
-    rc = hd->platform_ops->iotlb_flush_all(d);
+    rc = iommu_call(hd->platform_ops, iotlb_flush_all, d);
     if ( unlikely(rc) )
     {
         if ( !d->is_shutting_down && printk_ratelimit() )

diff --git a/xen/drivers/passthrough/pci.c b/xen/drivers/passthrough/pci.c
index 8108ed5f9a6dfc0a0dd59b77e38121bbe87666e1..061b20103fc387707d8c7410b6da09d0ed58b10f 100644 (file)
--- a/xen/drivers/passthrough/pci.c
+++ b/xen/drivers/passthrough/pci.c
@@ -1337,14 +1337,14 @@ int iommu_update_ire_from_msi(
     struct msi_desc *msi_desc, struct msi_msg *msg)
 {
     return iommu_intremap
-           ? iommu_get_ops()->update_ire_from_msi(msi_desc, msg) : 0;
+           ? iommu_call(&iommu_ops, update_ire_from_msi, msi_desc, msg) : 0;
 }
 
 void iommu_read_msi_from_ire(
     struct msi_desc *msi_desc, struct msi_msg *msg)
 {
     if ( iommu_intremap )
-        iommu_get_ops()->read_msi_from_ire(msi_desc, msg);
+        iommu_vcall(&iommu_ops, read_msi_from_ire, msi_desc, msg);
 }
 
 static int iommu_add_device(struct pci_dev *pdev)

diff --git a/xen/drivers/passthrough/x86/iommu.c b/xen/drivers/passthrough/x86/iommu.c
index 034ac903dd8013d7f12245c96e770ac4f3ae883e..0fa6dcc3fdef281ca33c49015ca76909aa446e24 100644 (file)
--- a/xen/drivers/passthrough/x86/iommu.c
+++ b/xen/drivers/passthrough/x86/iommu.c
@@ -61,14 +61,12 @@ int iommu_enable_x2apic(void)
 void iommu_update_ire_from_apic(
     unsigned int apic, unsigned int reg, unsigned int value)
 {
-    const struct iommu_ops *ops = iommu_get_ops();
-    ops->update_ire_from_apic(apic, reg, value);
+    iommu_vcall(&iommu_ops, update_ire_from_apic, apic, reg, value);
 }
 
 unsigned int iommu_read_apic_from_ire(unsigned int apic, unsigned int reg)
 {
-    const struct iommu_ops *ops = iommu_get_ops();
-    return ops->read_apic_from_ire(apic, reg);
+    return iommu_call(&iommu_ops, read_apic_from_ire, apic, reg);
 }
 
 int __init iommu_setup_hpet_msi(struct msi_desc *msi)

diff --git a/xen/include/asm-x86/iommu.h b/xen/include/asm-x86/iommu.h
index bbdb05f5f061abaf840df06632def0a9ef86e952..facf835adaffdac3df6fad4081713cefd45ff3de 100644 (file)
--- a/xen/include/asm-x86/iommu.h
+++ b/xen/include/asm-x86/iommu.h
@@ -59,6 +59,12 @@ struct arch_iommu
 
 extern struct iommu_ops iommu_ops;
 
+#ifdef NDEBUG
+# include <asm/alternative.h>
+# define iommu_call(ops, fn, args...)  alternative_call(iommu_ops.fn, ## args)
+# define iommu_vcall(ops, fn, args...) alternative_vcall(iommu_ops.fn, ## args)
+#endif
+
 static inline const struct iommu_ops *iommu_get_ops(void)
 {
     BUG_ON(!iommu_ops.init);

diff --git a/xen/include/xen/iommu.h b/xen/include/xen/iommu.h
index 5d3c1619c4d5048e8b1620ad7138867f60107c52..48f87480a700b78941becaaf3efb56f551bf0d57 100644 (file)
--- a/xen/include/xen/iommu.h
+++ b/xen/include/xen/iommu.h
@@ -244,6 +244,11 @@ struct iommu_ops {
 
 #include <asm/iommu.h>
 
+#ifndef iommu_call
+# define iommu_call(ops, fn, args...) ((ops)->fn(args))
+# define iommu_vcall iommu_call
+#endif
+
 enum iommu_status
 {
     IOMMU_STATUS_disabled,