nested vmx: optimize for bulk access of virtual VMCS

After we use the VMREAD/VMWRITE to build up the virtual VMCS, each
access to the virtual VMCS needs two VMPTRLD and one VMCLEAR to
switch the environment, which might be an overhead to performance.
This commit tries to handle multiple virtual VMCS access together
to improve the performance.

Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
Acked-by Eddie Dong <eddie.dong@intel.com>
Committed-by: Jan Beulich <jbeulich@suse.com>

diff --git a/xen/arch/x86/hvm/vmx/vmcs.c b/xen/arch/x86/hvm/vmx/vmcs.c
index 82a8d913c3434b753ee981472cfae13e19b6ab4c..f89ea930a9836af8b67b0d7206f808e7179e6f78 100644 (file)
--- a/xen/arch/x86/hvm/vmx/vmcs.c
+++ b/xen/arch/x86/hvm/vmx/vmcs.c
@@ -31,6 +31,7 @@
 #include <asm/hvm/io.h>
 #include <asm/hvm/support.h>
 #include <asm/hvm/vmx/vmx.h>
+#include <asm/hvm/vmx/vvmx.h>
 #include <asm/hvm/vmx/vmcs.h>
 #include <asm/flushtlb.h>
 #include <xen/event.h>
@@ -423,6 +424,13 @@ static void vmx_load_vmcs(struct vcpu *v)
 
 int vmx_cpu_up_prepare(unsigned int cpu)
 {
+    /*
+     * If nvmx_cpu_up_prepare() failed, do not return failure and just fallback
+     * to legacy mode for vvmcs synchronization.
+     */
+    if ( nvmx_cpu_up_prepare(cpu) != 0 )
+        printk("CPU%d: Could not allocate virtual VMCS buffer.\n", cpu);
+
     if ( per_cpu(vmxon_region, cpu) != NULL )
         return 0;
 
@@ -431,6 +439,7 @@ int vmx_cpu_up_prepare(unsigned int cpu)
         return 0;
 
     printk("CPU%d: Could not allocate host VMCS\n", cpu);
+    nvmx_cpu_dead(cpu);
     return -ENOMEM;
 }
 
@@ -438,6 +447,7 @@ void vmx_cpu_dead(unsigned int cpu)
 {
     vmx_free_vmcs(per_cpu(vmxon_region, cpu));
     per_cpu(vmxon_region, cpu) = NULL;
+    nvmx_cpu_dead(cpu);
 }
 
 int vmx_cpu_up(void)

diff --git a/xen/arch/x86/hvm/vmx/vvmx.c b/xen/arch/x86/hvm/vmx/vvmx.c
index 18d801f02db6e9534f743498c15f80c1f522c406..4a93785c1398c02bbf03f178ff81290b36b38b40 100644 (file)
--- a/xen/arch/x86/hvm/vmx/vvmx.c
+++ b/xen/arch/x86/hvm/vmx/vvmx.c
@@ -28,8 +28,31 @@
 #include <asm/hvm/vmx/vvmx.h>
 #include <asm/hvm/nestedhvm.h>
 
+static DEFINE_PER_CPU(u64 *, vvmcs_buf);
+
 static void nvmx_purge_vvmcs(struct vcpu *v);
 
+#define VMCS_BUF_SIZE 100
+
+int nvmx_cpu_up_prepare(unsigned int cpu)
+{
+    if ( per_cpu(vvmcs_buf, cpu) != NULL )
+        return 0;
+
+    per_cpu(vvmcs_buf, cpu) = xzalloc_array(u64, VMCS_BUF_SIZE);
+
+    if ( per_cpu(vvmcs_buf, cpu) != NULL )
+        return 0;
+
+    return -ENOMEM;
+}
+
+void nvmx_cpu_dead(unsigned int cpu)
+{
+    xfree(per_cpu(vvmcs_buf, cpu));
+    per_cpu(vvmcs_buf, cpu) = NULL;
+}
+
 int nvmx_vcpu_initialise(struct vcpu *v)
 {
     struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
@@ -834,6 +857,40 @@ static void vvmcs_to_shadow(void *vvmcs, unsigned int field)
     __vmwrite(field, value);
 }
 
+static void vvmcs_to_shadow_bulk(struct vcpu *v, unsigned int n,
+                                 const u16 *field)
+{
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    void *vvmcs = nvcpu->nv_vvmcx;
+    u64 *value = this_cpu(vvmcs_buf);
+    unsigned int i;
+
+    if ( !cpu_has_vmx_vmcs_shadowing )
+        goto fallback;
+
+    if ( !value || n > VMCS_BUF_SIZE )
+    {
+        gdprintk(XENLOG_DEBUG, "vmcs sync fall back to non-bulk mode, \
+                 buffer: %p, buffer size: %d, fields number: %d.\n",
+                 value, VMCS_BUF_SIZE, n);
+        goto fallback;
+    }
+
+    virtual_vmcs_enter(vvmcs);
+    for ( i = 0; i < n; i++ )
+        value[i] = __vmread(field[i]);
+    virtual_vmcs_exit(vvmcs);
+
+    for ( i = 0; i < n; i++ )
+        __vmwrite(field[i], value[i]);
+
+    return;
+
+fallback:
+    for ( i = 0; i < n; i++ )
+        vvmcs_to_shadow(vvmcs, field[i]);
+}
+
 static void shadow_to_vvmcs(void *vvmcs, unsigned int field)
 {
     u64 value;
@@ -844,6 +901,40 @@ static void shadow_to_vvmcs(void *vvmcs, unsigned int field)
         __set_vvmcs(vvmcs, field, value);
 }
 
+static void shadow_to_vvmcs_bulk(struct vcpu *v, unsigned int n,
+                                 const u16 *field)
+{
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    void *vvmcs = nvcpu->nv_vvmcx;
+    u64 *value = this_cpu(vvmcs_buf);
+    unsigned int i;
+
+    if ( !cpu_has_vmx_vmcs_shadowing )
+        goto fallback;
+
+    if ( !value || n > VMCS_BUF_SIZE )
+    {
+        gdprintk(XENLOG_DEBUG, "vmcs sync fall back to non-bulk mode, \
+                 buffer: %p, buffer size: %d, fields number: %d.\n",
+                 value, VMCS_BUF_SIZE, n);
+        goto fallback;
+    }
+
+    for ( i = 0; i < n; i++ )
+        value[i] = __vmread(field[i]);
+
+    virtual_vmcs_enter(vvmcs);
+    for ( i = 0; i < n; i++ )
+        __vmwrite(field[i], value[i]);
+    virtual_vmcs_exit(vvmcs);
+
+    return;
+
+fallback:
+    for ( i = 0; i < n; i++ )
+        shadow_to_vvmcs(vvmcs, field[i]);
+}
+
 static void load_shadow_control(struct vcpu *v)
 {
     /*
@@ -867,13 +958,18 @@ static void load_shadow_guest_state(struct vcpu *v)
 {
     struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
     void *vvmcs = nvcpu->nv_vvmcx;
-    int i;
     u32 control;
     u64 cr_gh_mask, cr_read_shadow;
 
+    static const u16 vmentry_fields[] = {
+        VM_ENTRY_INTR_INFO,
+        VM_ENTRY_EXCEPTION_ERROR_CODE,
+        VM_ENTRY_INSTRUCTION_LEN,
+    };
+
     /* vvmcs.gstate to shadow vmcs.gstate */
-    for ( i = 0; i < ARRAY_SIZE(vmcs_gstate_field); i++ )
-        vvmcs_to_shadow(vvmcs, vmcs_gstate_field[i]);
+    vvmcs_to_shadow_bulk(v, ARRAY_SIZE(vmcs_gstate_field),
+                         vmcs_gstate_field);
 
     hvm_set_cr0(__get_vvmcs(vvmcs, GUEST_CR0));
     hvm_set_cr4(__get_vvmcs(vvmcs, GUEST_CR4));
@@ -887,9 +983,7 @@ static void load_shadow_guest_state(struct vcpu *v)
 
     hvm_funcs.set_tsc_offset(v, v->arch.hvm_vcpu.cache_tsc_offset);
 
-    vvmcs_to_shadow(vvmcs, VM_ENTRY_INTR_INFO);
-    vvmcs_to_shadow(vvmcs, VM_ENTRY_EXCEPTION_ERROR_CODE);
-    vvmcs_to_shadow(vvmcs, VM_ENTRY_INSTRUCTION_LEN);
+    vvmcs_to_shadow_bulk(v, ARRAY_SIZE(vmentry_fields), vmentry_fields);
 
     /*
      * While emulate CR0 and CR4 for nested virtualization, set the CR0/CR4
@@ -909,10 +1003,13 @@ static void load_shadow_guest_state(struct vcpu *v)
     if ( nvmx_ept_enabled(v) && hvm_pae_enabled(v) &&
          (v->arch.hvm_vcpu.guest_efer & EFER_LMA) )
     {
-        vvmcs_to_shadow(vvmcs, GUEST_PDPTR0);
-        vvmcs_to_shadow(vvmcs, GUEST_PDPTR1);
-        vvmcs_to_shadow(vvmcs, GUEST_PDPTR2);
-        vvmcs_to_shadow(vvmcs, GUEST_PDPTR3);
+        static const u16 gpdptr_fields[] = {
+            GUEST_PDPTR0,
+            GUEST_PDPTR1,
+            GUEST_PDPTR2,
+            GUEST_PDPTR3,
+        };
+        vvmcs_to_shadow_bulk(v, ARRAY_SIZE(gpdptr_fields), gpdptr_fields);
     }
 
     /* TODO: CR3 target control */
@@ -1003,13 +1100,12 @@ static void virtual_vmentry(struct cpu_user_regs *regs)
 
 static void sync_vvmcs_guest_state(struct vcpu *v, struct cpu_user_regs *regs)
 {
-    int i;
     struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
     void *vvmcs = nvcpu->nv_vvmcx;
 
     /* copy shadow vmcs.gstate back to vvmcs.gstate */
-    for ( i = 0; i < ARRAY_SIZE(vmcs_gstate_field); i++ )
-        shadow_to_vvmcs(vvmcs, vmcs_gstate_field[i]);
+    shadow_to_vvmcs_bulk(v, ARRAY_SIZE(vmcs_gstate_field),
+                         vmcs_gstate_field);
     /* RIP, RSP are in user regs */
     __set_vvmcs(vvmcs, GUEST_RIP, regs->eip);
     __set_vvmcs(vvmcs, GUEST_RSP, regs->esp);
@@ -1021,13 +1117,11 @@ static void sync_vvmcs_guest_state(struct vcpu *v, struct cpu_user_regs *regs)
 
 static void sync_vvmcs_ro(struct vcpu *v)
 {
-    int i;
     struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
     struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
     void *vvmcs = nvcpu->nv_vvmcx;
 
-    for ( i = 0; i < ARRAY_SIZE(vmcs_ro_field); i++ )
-        shadow_to_vvmcs(nvcpu->nv_vvmcx, vmcs_ro_field[i]);
+    shadow_to_vvmcs_bulk(v, ARRAY_SIZE(vmcs_ro_field), vmcs_ro_field);
 
     /* Adjust exit_reason/exit_qualifciation for violation case */
     if ( __get_vvmcs(vvmcs, VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION )

diff --git a/xen/include/asm-x86/hvm/vmx/vvmx.h b/xen/include/asm-x86/hvm/vmx/vvmx.h
index 73a67cc2c8406004dd9497b01a5e94faa5ffe904..3874525c1ac6334e7ff881bae7e1b697de2a8455 100644 (file)
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h
@@ -229,5 +229,7 @@ int nept_translate_l2ga(struct vcpu *v, paddr_t l2ga,
                         unsigned int *page_order, uint32_t rwx_acc,
                         unsigned long *l1gfn, uint8_t *p2m_acc,
                         uint64_t *exit_qual, uint32_t *exit_reason);
+int nvmx_cpu_up_prepare(unsigned int cpu);
+void nvmx_cpu_dead(unsigned int cpu);
 #endif /* __ASM_X86_HVM_VVMX_H__ */