xen/x86: use invpcid for flushing the TLB

If possible use the INVPCID instruction for flushing the TLB instead of
toggling cr4.pge for that purpose.

While at it remove the dependency on cr4.pge being required for mtrr
loading, as this will be required later anyway.

Add a command line option "invpcid" for controlling the use of
INVPCID (default to true).

Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown
index 29e84c4a18e4ff0a234da3833ace0704853c20f6..ac3b8b66817af383eafbcbed75db371764ddc1a8 100644 (file)
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1369,6 +1369,15 @@ Because responsibility for APIC setup is shared between Xen and the
 domain 0 kernel this option is automatically propagated to the domain
 0 command line.
 
+### invpcid (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+By default, Xen will use the INVPCID instruction for TLB management if
+it is available.  This option can be used to cause Xen to fall back to
+older mechanisms, which are generally slower.
+
 ### noirqbalance
 > `= <boolean>`
 

diff --git a/xen/arch/x86/cpu/mtrr/generic.c b/xen/arch/x86/cpu/mtrr/generic.c
index e9c0e5e059555448d5f5599ed8d1b964488c5d21..7ba0c3f0fe893cdc32a9ebecdb4dbd8b8a14ce83 100644 (file)
--- a/xen/arch/x86/cpu/mtrr/generic.c
+++ b/xen/arch/x86/cpu/mtrr/generic.c
@@ -5,6 +5,7 @@
 #include <xen/mm.h>
 #include <xen/stdbool.h>
 #include <asm/flushtlb.h>
+#include <asm/invpcid.h>
 #include <asm/io.h>
 #include <asm/mtrr.h>
 #include <asm/msr.h>
@@ -400,8 +401,10 @@ static DEFINE_SPINLOCK(set_atomicity_lock);
  * has been called.
  */
 
-static void prepare_set(void)
+static bool prepare_set(void)
 {
+       unsigned long cr4;
+
        /*  Note that this is not ideal, since the cache is only flushed/disabled
           for this CPU while the MTRRs are changed, but changing this requires
           more invasive changes to the way the kernel boots  */
@@ -412,18 +415,24 @@ static void prepare_set(void)
        write_cr0(read_cr0() | X86_CR0_CD);
        wbinvd();
 
-       /*  TLB flushing here relies on Xen always using CR4.PGE. */
-       BUILD_BUG_ON(!(XEN_MINIMAL_CR4 & X86_CR4_PGE));
-       write_cr4(read_cr4() & ~X86_CR4_PGE);
+       cr4 = read_cr4();
+       if (cr4 & X86_CR4_PGE)
+               write_cr4(cr4 & ~X86_CR4_PGE);
+       else if (use_invpcid)
+               invpcid_flush_all();
+       else
+               write_cr3(read_cr3());
 
        /*  Save MTRR state */
        rdmsrl(MSR_MTRRdefType, deftype);
 
        /*  Disable MTRRs, and set the default type to uncached  */
        mtrr_wrmsr(MSR_MTRRdefType, deftype & ~0xcff);
+
+       return cr4 & X86_CR4_PGE;
 }
 
-static void post_set(void)
+static void post_set(bool pge)
 {
        /* Intel (P6) standard MTRRs */
        mtrr_wrmsr(MSR_MTRRdefType, deftype);
@@ -432,7 +441,12 @@ static void post_set(void)
        write_cr0(read_cr0() & ~X86_CR0_CD);
 
        /*  Reenable CR4.PGE (also flushes the TLB) */
-       write_cr4(read_cr4() | X86_CR4_PGE);
+       if (pge)
+               write_cr4(read_cr4() | X86_CR4_PGE);
+       else if (use_invpcid)
+               invpcid_flush_all();
+       else
+               write_cr3(read_cr3());
 
        spin_unlock(&set_atomicity_lock);
 }
@@ -441,14 +455,15 @@ static void generic_set_all(void)
 {
        unsigned long mask, count;
        unsigned long flags;
+       bool pge;
 
        local_irq_save(flags);
-       prepare_set();
+       pge = prepare_set();
 
        /* Actually set the state */
        mask = set_mtrr_state();
 
-       post_set();
+       post_set(pge);
        local_irq_restore(flags);
 
        /*  Use the atomic bitops to update the global mask  */
@@ -457,7 +472,6 @@ static void generic_set_all(void)
                        set_bit(count, &smp_changes_mask);
                mask >>= 1;
        }
-       
 }
 
 static void generic_set_mtrr(unsigned int reg, unsigned long base,
@@ -474,11 +488,12 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,
 {
        unsigned long flags;
        struct mtrr_var_range *vr;
+       bool pge;
 
        vr = &mtrr_state.var_ranges[reg];
 
        local_irq_save(flags);
-       prepare_set();
+       pge = prepare_set();
 
        if (size == 0) {
                /* The invalid bit is kept in the mask, so we simply clear the
@@ -499,7 +514,7 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,
                mtrr_wrmsr(MSR_IA32_MTRR_PHYSMASK(reg), vr->mask);
        }
 
-       post_set();
+       post_set(pge);
        local_irq_restore(flags);
 }
 

diff --git a/xen/arch/x86/flushtlb.c b/xen/arch/x86/flushtlb.c
index 1af9221607bf7587802fc65d9a21412d7ff4b5ac..b6817f18170f2640aed93517c90d59727df613e3 100644 (file)
--- a/xen/arch/x86/flushtlb.c
+++ b/xen/arch/x86/flushtlb.c
@@ -11,6 +11,7 @@
 #include <xen/smp.h>
 #include <xen/softirq.h>
 #include <asm/flushtlb.h>
+#include <asm/invpcid.h>
 #include <asm/page.h>
 
 /* Debug builds: Wrap frequently to stress-test the wrap logic. */
@@ -72,6 +73,23 @@ static void post_flush(u32 t)
     this_cpu(tlbflush_time) = t;
 }
 
+static void do_tlb_flush(void)
+{
+    u32 t = pre_flush();
+
+    if ( use_invpcid )
+        invpcid_flush_all();
+    else
+    {
+        unsigned long cr4 = read_cr4();
+
+        write_cr4(cr4 ^ X86_CR4_PGE);
+        write_cr4(cr4);
+    }
+
+    post_flush(t);
+}
+
 void switch_cr3(unsigned long cr3)
 {
     unsigned long flags, cr4;
@@ -119,16 +137,7 @@ unsigned int flush_area_local(const void *va, unsigned int flags)
                            : : "m" (*(const char *)(va)) : "memory" );
         }
         else
-        {
-            u32 t = pre_flush();
-            unsigned long cr4 = read_cr4();
-
-            write_cr4(cr4 & ~X86_CR4_PGE);
-            barrier();
-            write_cr4(cr4);
-
-            post_flush(t);
-        }
+            do_tlb_flush();
     }
 
     if ( flags & FLUSH_CACHE )

diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index f803980b97e7ffe68256a9a58cd981cf975d7ef9..164c42cbf197a0936d09ba2f2e79fb375f488129 100644 (file)
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -62,6 +62,11 @@ boolean_param("nosmp", opt_nosmp);
 static unsigned int __initdata max_cpus;
 integer_param("maxcpus", max_cpus);
 
+/* opt_invpcid: If false, don't use INVPCID instruction even if available. */
+static bool __initdata opt_invpcid = true;
+boolean_param("invpcid", opt_invpcid);
+bool __read_mostly use_invpcid;
+
 unsigned long __read_mostly cr4_pv32_mask;
 
 /* **** Linux config option: propagated to domain0. */
@@ -1546,6 +1551,9 @@ void __init noreturn __start_xen(unsigned long mbi_p)
     if ( cpu_has_fsgsbase )
         set_in_cr4(X86_CR4_FSGSBASE);
 
+    if ( opt_invpcid && cpu_has_invpcid )
+        use_invpcid = true;
+
     init_speculation_mitigations();
 
     init_idle_domain();

diff --git a/xen/include/asm-x86/invpcid.h b/xen/include/asm-x86/invpcid.h
index b46624a865c78d01bcaf1b7d82c8517b6da694ab..edd8b68706d24eed51a9495afd7b1bcdff1af493 100644 (file)
--- a/xen/include/asm-x86/invpcid.h
+++ b/xen/include/asm-x86/invpcid.h
@@ -3,6 +3,8 @@
 
 #include <xen/types.h>
 
+extern bool use_invpcid;
+
 #define INVPCID_TYPE_INDIV_ADDR      0
 #define INVPCID_TYPE_SINGLE_CTXT     1
 #define INVPCID_TYPE_ALL_INCL_GLOBAL 2