x86/xpti: avoid copying L4 page table contents when possible

For mitigation of Meltdown the current L4 page table is copied to the
cpu local root page table each time a 64 bit pv guest is entered.

Copying can be avoided in cases where the guest L4 page table hasn't
been modified while running the hypervisor, e.g. when handling
interrupts or any hypercall not modifying the L4 page table or %cr3.

So add a per-cpu flag indicating whether the copying should be
performed and set that flag only when loading a new %cr3 or modifying
the L4 page table.  This includes synchronization of the cpu local
root page table with other cpus, so add a special synchronization flag
for that case.

A simple performance check (compiling the hypervisor via "make -j 4")
in dom0 with 4 vcpus shows a significant improvement:

- real time drops from 112 seconds to 103 seconds
- system time drops from 142 seconds to 131 seconds

Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

diff --git a/xen/arch/x86/flushtlb.c b/xen/arch/x86/flushtlb.c
index 7d5a81576fc48c809dd501708ffa7a01e1bc1411..96e5003d797fe91da11eedd513c4f287725fb125 100644 (file)
--- a/xen/arch/x86/flushtlb.c
+++ b/xen/arch/x86/flushtlb.c
@@ -9,6 +9,7 @@
 
 #include <xen/config.h>
 #include <xen/sched.h>
+#include <xen/smp.h>
 #include <xen/softirq.h>
 #include <asm/flushtlb.h>
 #include <asm/page.h>
@@ -151,4 +152,7 @@ void flush_area_local(const void *va, unsigned int flags)
     }
 
     local_irq_restore(irqfl);
+
+    if ( flags & FLUSH_ROOT_PGTBL )
+        get_cpu_info()->root_pgt_changed = 1;
 }

diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index dfa1934412fcb545b61eca786554abdbc560923c..87db8a815861187ce47ae8b4f0c91e82c39ac57b 100644 (file)
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -488,6 +488,7 @@ void make_cr3(struct vcpu *v, unsigned long mfn)
 
 void write_ptbase(struct vcpu *v)
 {
+    get_cpu_info()->root_pgt_changed = 1;
     write_cr3(v->arch.cr3);
 }
 
@@ -3911,9 +3912,13 @@ long do_mmu_update(
                 case PGT_l4_page_table:
                     rc = mod_l4_entry(va, l4e_from_intpte(req.val), mfn,
                                       cmd == MMU_PT_UPDATE_PRESERVE_AD, v);
-                    if ( !rc )
-                        sync_guest = !!this_cpu(root_pgt);
-                break;
+                    if ( !rc && this_cpu(root_pgt) )
+                    {
+                        sync_guest = 1;
+                        if ( pagetable_get_pfn(curr->arch.guest_table) == mfn )
+                            get_cpu_info()->root_pgt_changed = 1;
+                    }
+                    break;
                 case PGT_writable_page:
                     perfc_incr(writable_mmu_updates);
                     if ( paging_write_guest_entry(v, va, req.val, _mfn(mfn)) )
@@ -4019,14 +4024,10 @@ long do_mmu_update(
     {
         /*
          * Force other vCPU-s of the affected guest to pick up L4 entry
-         * changes (if any). Issue a flush IPI with empty operation mask to
-         * facilitate this (including ourselves waiting for the IPI to
-         * actually have arrived). Utilize the fact that FLUSH_VA_VALID is
-         * meaningless without FLUSH_CACHE, but will allow to pass the no-op
-         * check in flush_area_mask().
+         * changes (if any).
          */
         flush_area_mask(pt_owner->domain_dirty_cpumask,
-                        ZERO_BLOCK_PTR, FLUSH_VA_VALID);
+                        ZERO_BLOCK_PTR, FLUSH_ROOT_PGTBL);
     }
 
     perfc_add(num_page_updates, i);

diff --git a/xen/arch/x86/mm/shadow/multi.c b/xen/arch/x86/mm/shadow/multi.c
index 117114b3a013d7f5cf619e3a5521bae4c7313964..d155092e226e822796d12660e267fa4d559e6526 100644 (file)
--- a/xen/arch/x86/mm/shadow/multi.c
+++ b/xen/arch/x86/mm/shadow/multi.c
@@ -923,6 +923,8 @@ static int shadow_set_l4e(struct domain *d,
 
     /* Write the new entry */
     shadow_write_entries(sl4e, &new_sl4e, 1, sl4mfn);
+    flush_root_pgtbl_domain(d);
+
     flags |= SHADOW_SET_CHANGED;
 
     if ( shadow_l4e_get_flags(old_sl4e) & _PAGE_PRESENT )
@@ -937,6 +939,7 @@ static int shadow_set_l4e(struct domain *d,
         }
         sh_put_ref(d, osl3mfn, paddr);
     }
+
     return flags;
 }
 

diff --git a/xen/arch/x86/smp.c b/xen/arch/x86/smp.c
index 8caa0bce3383fc1da87cef548ad23e29b18a88da..b57534659669f9e2801c929fa07c02c99d86dbca 100644 (file)
--- a/xen/arch/x86/smp.c
+++ b/xen/arch/x86/smp.c
@@ -213,7 +213,7 @@ void invalidate_interrupt(struct cpu_user_regs *regs)
     ack_APIC_irq();
     perfc_incr(ipis);
     if ( !__sync_local_execstate() ||
-         (flush_flags & (FLUSH_TLB_GLOBAL | FLUSH_CACHE)) )
+         (flush_flags & (FLUSH_TLB_GLOBAL | FLUSH_CACHE | FLUSH_ROOT_PGTBL)) )
         flush_area_local(flush_va, flush_flags);
     cpumask_clear_cpu(smp_processor_id(), &flush_cpumask);
 }

diff --git a/xen/arch/x86/x86_64/asm-offsets.c b/xen/arch/x86/x86_64/asm-offsets.c
index 85a8aec2f5e5ec84a8b8c1f762299c8c03e27570..b72611f0d2c571f0850260689cf6c11788d6dc32 100644 (file)
--- a/xen/arch/x86/x86_64/asm-offsets.c
+++ b/xen/arch/x86/x86_64/asm-offsets.c
@@ -142,6 +142,7 @@ void __dummy__(void)
     OFFSET(CPUINFO_shadow_spec_ctrl, struct cpu_info, shadow_spec_ctrl);
     OFFSET(CPUINFO_xen_spec_ctrl, struct cpu_info, xen_spec_ctrl);
     OFFSET(CPUINFO_spec_ctrl_flags, struct cpu_info, spec_ctrl_flags);
+    OFFSET(CPUINFO_root_pgt_changed, struct cpu_info, root_pgt_changed);
     DEFINE(CPUINFO_sizeof, sizeof(struct cpu_info));
     BLANK();
 

diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S
index 8cecfd407ed40da3b7d6f5fb4eb620c5142a1879..ad2c4b8fa57a61d40c90b8b1f701b5b14d8cf1b9 100644 (file)
--- a/xen/arch/x86/x86_64/entry.S
+++ b/xen/arch/x86/x86_64/entry.S
@@ -44,11 +44,15 @@ restore_all_guest:
         mov   %cr3, %r9
         GET_STACK_BASE(%rdx)
         mov   STACK_CPUINFO_FIELD(pv_cr3)(%rdx), %rdi
+        test  %rdi, %rdi
+        jz    .Lrag_keep_cr3
+        mov   %rdi, %rax
+        cmpb  $0, STACK_CPUINFO_FIELD(root_pgt_changed)(%rdx)
+        je    .Lrag_copy_done
+        movb  $0, STACK_CPUINFO_FIELD(root_pgt_changed)(%rdx)
         movabs $PADDR_MASK & PAGE_MASK, %rsi
         movabs $DIRECTMAP_VIRT_START, %rcx
-        mov   %rdi, %rax
         and   %rsi, %rdi
-        jz    .Lrag_keep_cr3
         and   %r9, %rsi
         add   %rcx, %rdi
         add   %rcx, %rsi
@@ -63,6 +67,7 @@ restore_all_guest:
         sub   $(ROOT_PAGETABLE_FIRST_XEN_SLOT - \
                 ROOT_PAGETABLE_LAST_XEN_SLOT - 1) * 8, %rdi
         rep movsq
+.Lrag_copy_done:
         mov   %r9, STACK_CPUINFO_FIELD(xen_cr3)(%rdx)
         write_cr3 rax, rdi, rsi
 .Lrag_keep_cr3:

diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c
index ffab05c3899e2ca719a3914b59516456a1b54d6c..717e51cae3c1b637416697755b765651da36481c 100644 (file)
--- a/xen/arch/x86/x86_64/traps.c
+++ b/xen/arch/x86/x86_64/traps.c
@@ -263,6 +263,8 @@ void toggle_guest_mode(struct vcpu *v)
     v->arch.flags ^= TF_kernel_mode;
     asm volatile ( "swapgs" );
     update_cr3(v);
+    get_cpu_info()->root_pgt_changed = 1;
+
     /* Don't flush user global mappings from the TLB. Don't tick TLB clock. */
     asm volatile ( "mov %0, %%cr3" : : "r" (v->arch.cr3) : "memory" );
 

diff --git a/xen/include/asm-x86/current.h b/xen/include/asm-x86/current.h
index 04bc7605fa010c36de7a897999cee08ce16babf1..402cff0648041b78d0219b9d21eadc686f2c0100 100644 (file)
--- a/xen/include/asm-x86/current.h
+++ b/xen/include/asm-x86/current.h
@@ -60,6 +60,14 @@ struct cpu_info {
     uint8_t      xen_spec_ctrl;
     uint8_t      spec_ctrl_flags;
 
+    /*
+     * The following field controls copying of the L4 page table of 64-bit
+     * PV guests to the per-cpu root page table on entering the guest context.
+     * If set the L4 page table is being copied to the root page table and
+     * the field will be reset.
+     */
+    bool_t       root_pgt_changed;
+
     unsigned long __pad;
     /* get_stack_bottom() must be 16-byte aligned */
 };

diff --git a/xen/include/asm-x86/flushtlb.h b/xen/include/asm-x86/flushtlb.h
index f1389daa6fb9d213959fc30660b106428d1c429a..ecc944c717ac18f9da8ccc689a0b51a04972833b 100644 (file)
--- a/xen/include/asm-x86/flushtlb.h
+++ b/xen/include/asm-x86/flushtlb.h
@@ -101,6 +101,8 @@ void write_cr3(unsigned long cr3);
 #define FLUSH_CACHE      0x400
  /* VA for the flush has a valid mapping */
 #define FLUSH_VA_VALID   0x800
+ /* Flush the per-cpu root page table */
+#define FLUSH_ROOT_PGTBL 0x2000
 
 /* Flush local TLBs/caches. */
 void flush_area_local(const void *va, unsigned int flags);
@@ -132,6 +134,12 @@ void flush_area_mask(const cpumask_t *, const void *va, unsigned int flags);
 #define flush_tlb_one_all(v)                    \
     flush_tlb_one_mask(&cpu_online_map, v)
 
+#define flush_root_pgtbl_domain(d)                                       \
+{                                                                        \
+    if ( this_cpu(root_pgt) && is_pv_domain(d) && !is_pv_32bit_domain(d) ) \
+        flush_mask((d)->domain_dirty_cpumask, FLUSH_ROOT_PGTBL);         \
+}
+
 static inline void flush_page_to_ram(unsigned long mfn) {}
 static inline int invalidate_dcache_va_range(const void *p,
                                              unsigned long size)