ia64/xen-unstable

changeset 17416:0553004fa328

x86, vmx: Enable VPID (Virtual Processor Identification)

Allows TLB entries to be retained across VM entry and VM exit, and Xen
can now identify distinct address spaces through a new
virtual-processor ID (VPID) field of the VMCS.

Signed-off-by: Xin Li <xin.b.li@intel.com>
Signed-off-by: Jun Nakajima <jun.nakajima@intel.com>
Signed-off-by: Xiaohui Xin <Xiaohui.xin@intel.com>
Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Wed Apr 09 14:34:49 2008 +0100 (2008-04-09)
parents 3a213b0e1ac0
children e64d17985df5
files xen/arch/x86/hvm/vmx/vmcs.c xen/arch/x86/hvm/vmx/vmx.c xen/include/asm-x86/hvm/vmx/vmcs.h xen/include/asm-x86/hvm/vmx/vmx.h
line diff
     1.1 --- a/xen/arch/x86/hvm/vmx/vmcs.c	Wed Apr 09 14:10:32 2008 +0100
     1.2 +++ b/xen/arch/x86/hvm/vmx/vmcs.c	Wed Apr 09 14:34:49 2008 +0100
     1.3 @@ -38,6 +38,9 @@
     1.4  #include <asm/shadow.h>
     1.5  #include <asm/tboot.h>
     1.6  
     1.7 +static int opt_vpid_enabled = 1;
     1.8 +boolean_param("vpid", opt_vpid_enabled);
     1.9 +
    1.10  /* Dynamic (run-time adjusted) execution control flags. */
    1.11  u32 vmx_pin_based_exec_control __read_mostly;
    1.12  u32 vmx_cpu_based_exec_control __read_mostly;
    1.13 @@ -111,6 +114,8 @@ static void vmx_init_vmcs_config(void)
    1.14          opt = (SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
    1.15                 SECONDARY_EXEC_WBINVD_EXITING |
    1.16                 SECONDARY_EXEC_ENABLE_EPT);
    1.17 +        if ( opt_vpid_enabled )
    1.18 +            opt |= SECONDARY_EXEC_ENABLE_VPID;
    1.19          _vmx_secondary_exec_control = adjust_vmx_controls(
    1.20              min, opt, MSR_IA32_VMX_PROCBASED_CTLS2);
    1.21      }
    1.22 @@ -317,6 +322,8 @@ int vmx_cpu_up(void)
    1.23  
    1.24      ept_sync_all();
    1.25  
    1.26 +    vpid_sync_all();
    1.27 +
    1.28      return 1;
    1.29  }
    1.30  
    1.31 @@ -629,6 +636,13 @@ static int construct_vmcs(struct vcpu *v
    1.32  #endif
    1.33      }
    1.34  
    1.35 +    if ( cpu_has_vmx_vpid )
    1.36 +    {
    1.37 +        v->arch.hvm_vmx.vpid =
    1.38 +            v->domain->arch.hvm_domain.vmx.vpid_base + v->vcpu_id;
    1.39 +        __vmwrite(VIRTUAL_PROCESSOR_ID, v->arch.hvm_vmx.vpid);
    1.40 +    }
    1.41 +
    1.42      vmx_vmcs_exit(v);
    1.43  
    1.44      paging_update_paging_modes(v); /* will update HOST & GUEST_CR3 as reqd */
    1.45 @@ -822,6 +836,7 @@ void vmx_do_resume(struct vcpu *v)
    1.46          vmx_load_vmcs(v);
    1.47          hvm_migrate_timers(v);
    1.48          vmx_set_host_env(v);
    1.49 +        vpid_sync_vcpu_all(v);
    1.50      }
    1.51  
    1.52      debug_state = v->domain->debugger_attached;
    1.53 @@ -976,6 +991,8 @@ void vmcs_dump_vcpu(struct vcpu *v)
    1.54             (uint32_t)vmr(TPR_THRESHOLD));
    1.55      printk("EPT pointer = 0x%08x%08x\n",
    1.56             (uint32_t)vmr(EPT_POINTER_HIGH), (uint32_t)vmr(EPT_POINTER));
    1.57 +    printk("Virtual processor ID = 0x%04x\n",
    1.58 +           (uint32_t)vmr(VIRTUAL_PROCESSOR_ID));
    1.59  
    1.60      vmx_vmcs_exit(v);
    1.61  }
     2.1 --- a/xen/arch/x86/hvm/vmx/vmx.c	Wed Apr 09 14:10:32 2008 +0100
     2.2 +++ b/xen/arch/x86/hvm/vmx/vmx.c	Wed Apr 09 14:34:49 2008 +0100
     2.3 @@ -57,6 +57,8 @@ static void vmx_ctxt_switch_to(struct vc
     2.4  
     2.5  static int  vmx_alloc_vlapic_mapping(struct domain *d);
     2.6  static void vmx_free_vlapic_mapping(struct domain *d);
     2.7 +static int  vmx_alloc_vpid(struct domain *d);
     2.8 +static void vmx_free_vpid(struct domain *d);
     2.9  static void vmx_install_vlapic_mapping(struct vcpu *v);
    2.10  static void vmx_update_guest_cr(struct vcpu *v, unsigned int cr);
    2.11  static void vmx_update_guest_efer(struct vcpu *v);
    2.12 @@ -71,18 +73,30 @@ static void vmx_invlpg_intercept(unsigne
    2.13  
    2.14  static int vmx_domain_initialise(struct domain *d)
    2.15  {
    2.16 +    int rc;
    2.17 +
    2.18      d->arch.hvm_domain.vmx.ept_control.etmt = EPT_DEFAULT_MT;
    2.19      d->arch.hvm_domain.vmx.ept_control.gaw  = EPT_DEFAULT_GAW;
    2.20      d->arch.hvm_domain.vmx.ept_control.asr  =
    2.21          pagetable_get_pfn(d->arch.phys_table);
    2.22  
    2.23 -    return vmx_alloc_vlapic_mapping(d);
    2.24 +    if ( (rc = vmx_alloc_vpid(d)) != 0 )
    2.25 +        return rc;
    2.26 +
    2.27 +    if ( (rc = vmx_alloc_vlapic_mapping(d)) != 0 )
    2.28 +    {
    2.29 +        vmx_free_vpid(d);
    2.30 +        return rc;
    2.31 +    }
    2.32 +
    2.33 +    return 0;
    2.34  }
    2.35  
    2.36  static void vmx_domain_destroy(struct domain *d)
    2.37  {
    2.38      ept_sync_domain(d);
    2.39      vmx_free_vlapic_mapping(d);
    2.40 +    vmx_free_vpid(d);
    2.41  }
    2.42  
    2.43  static int vmx_vcpu_initialise(struct vcpu *v)
    2.44 @@ -1024,6 +1038,7 @@ static void vmx_update_guest_cr(struct v
    2.45          }
    2.46   
    2.47          __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr[3]);
    2.48 +        vpid_sync_vcpu_all(v);
    2.49          break;
    2.50      case 4:
    2.51          v->arch.hvm_vcpu.hw_cr[4] = HVM_CR4_HOST_MASK;
    2.52 @@ -1069,9 +1084,15 @@ static void vmx_update_guest_efer(struct
    2.53  
    2.54  static void vmx_flush_guest_tlbs(void)
    2.55  {
    2.56 -    /* No tagged TLB support on VMX yet.  The fact that we're in Xen
    2.57 -     * at all means any guest will have a clean TLB when it's next run,
    2.58 -     * because VMRESUME will flush it for us. */
    2.59 +    /*
    2.60 +     * If VPID (i.e. tagged TLB support) is not enabled, the fact that
    2.61 +     * we're in Xen at all means any guest will have a clean TLB when
    2.62 +     * it's next run, because VMRESUME will flush it for us.
    2.63 +     *
    2.64 +     * If enabled, we invalidate all translations associated with all
    2.65 +     * VPID values.
    2.66 +     */
    2.67 +    vpid_sync_all();
    2.68  }
    2.69  
    2.70  static void __ept_sync_domain(void *info)
    2.71 @@ -1202,6 +1223,9 @@ static struct hvm_function_table vmx_fun
    2.72      .invlpg_intercept     = vmx_invlpg_intercept
    2.73  };
    2.74  
    2.75 +static unsigned long *vpid_bitmap;
    2.76 +#define VPID_BITMAP_SIZE ((1u << VMCS_VPID_WIDTH) / MAX_VIRT_CPUS)
    2.77 +
    2.78  void start_vmx(void)
    2.79  {
    2.80      static int bootstrapped;
    2.81 @@ -1241,6 +1265,19 @@ void start_vmx(void)
    2.82          vmx_function_table.hap_supported = 1;
    2.83      }
    2.84  
    2.85 +    if ( cpu_has_vmx_vpid )
    2.86 +    {
    2.87 +        printk("VMX: VPID is available.\n");
    2.88 +
    2.89 +        vpid_bitmap = xmalloc_array(
    2.90 +            unsigned long, BITS_TO_LONGS(VPID_BITMAP_SIZE));
    2.91 +        BUG_ON(vpid_bitmap == NULL);
    2.92 +        memset(vpid_bitmap, 0, BITS_TO_LONGS(VPID_BITMAP_SIZE) * sizeof(long));
    2.93 +
    2.94 +        /* VPID 0 is used by VMX root mode (the hypervisor). */
    2.95 +        __set_bit(0, vpid_bitmap);
    2.96 +    }
    2.97 +
    2.98      setup_vmcs_dump();
    2.99  
   2.100      hvm_enable(&vmx_function_table);
   2.101 @@ -1755,6 +1792,35 @@ static void vmx_free_vlapic_mapping(stru
   2.102          free_xenheap_page(mfn_to_virt(mfn));
   2.103  }
   2.104  
   2.105 +static int vmx_alloc_vpid(struct domain *d)
   2.106 +{
   2.107 +    int idx;
   2.108 +
   2.109 +    if ( !cpu_has_vmx_vpid )
   2.110 +        return 0;
   2.111 +
   2.112 +    do {
   2.113 +        idx = find_first_zero_bit(vpid_bitmap, VPID_BITMAP_SIZE);
   2.114 +        if ( idx >= VPID_BITMAP_SIZE )
   2.115 +        {
   2.116 +            dprintk(XENLOG_WARNING, "VMX VPID space exhausted.\n");
   2.117 +            return -EBUSY;
   2.118 +        }
   2.119 +    }
   2.120 +    while ( test_and_set_bit(idx, vpid_bitmap) );
   2.121 +
   2.122 +    d->arch.hvm_domain.vmx.vpid_base = idx * MAX_VIRT_CPUS;
   2.123 +    return 0;
   2.124 +}
   2.125 +
   2.126 +static void vmx_free_vpid(struct domain *d)
   2.127 +{
   2.128 +    if ( !cpu_has_vmx_vpid )
   2.129 +        return;
   2.130 +
   2.131 +    clear_bit(d->arch.hvm_domain.vmx.vpid_base / MAX_VIRT_CPUS, vpid_bitmap);
   2.132 +}
   2.133 +
   2.134  static void vmx_install_vlapic_mapping(struct vcpu *v)
   2.135  {
   2.136      paddr_t virt_page_ma, apic_page_ma;
     3.1 --- a/xen/include/asm-x86/hvm/vmx/vmcs.h	Wed Apr 09 14:10:32 2008 +0100
     3.2 +++ b/xen/include/asm-x86/hvm/vmx/vmcs.h	Wed Apr 09 14:34:49 2008 +0100
     3.3 @@ -58,7 +58,7 @@ struct vmx_msr_state {
     3.4  
     3.5  struct vmx_domain {
     3.6      unsigned long apic_access_mfn;
     3.7 -
     3.8 +    unsigned long vpid_base;
     3.9      union {
    3.10          struct {
    3.11              u64 etmt :3,
    3.12 @@ -90,6 +90,8 @@ struct arch_vmx_struct {
    3.13      u32                  exec_control;
    3.14      u32                  secondary_exec_control;
    3.15  
    3.16 +    u16                  vpid;
    3.17 +
    3.18      /* PMU */
    3.19      struct vpmu_struct   vpmu;
    3.20  
    3.21 @@ -157,6 +159,7 @@ extern u32 vmx_vmentry_control;
    3.22  
    3.23  #define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001
    3.24  #define SECONDARY_EXEC_ENABLE_EPT               0x00000002
    3.25 +#define SECONDARY_EXEC_ENABLE_VPID              0x00000020
    3.26  #define SECONDARY_EXEC_WBINVD_EXITING           0x00000040
    3.27  extern u32 vmx_secondary_exec_control;
    3.28  
    3.29 @@ -176,6 +179,8 @@ extern bool_t cpu_has_vmx_ins_outs_instr
    3.30      (vmx_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
    3.31  #define cpu_has_vmx_ept \
    3.32      (vmx_secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT)
    3.33 +#define cpu_has_vmx_vpid \
    3.34 +    (vmx_secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
    3.35  
    3.36  /* GUEST_INTERRUPTIBILITY_INFO flags. */
    3.37  #define VMX_INTR_SHADOW_STI             0x00000001
    3.38 @@ -185,6 +190,7 @@ extern bool_t cpu_has_vmx_ins_outs_instr
    3.39  
    3.40  /* VMCS field encodings. */
    3.41  enum vmcs_field {
    3.42 +    VIRTUAL_PROCESSOR_ID            = 0x00000000,
    3.43      GUEST_ES_SELECTOR               = 0x00000800,
    3.44      GUEST_CS_SELECTOR               = 0x00000802,
    3.45      GUEST_SS_SELECTOR               = 0x00000804,
    3.46 @@ -324,6 +330,8 @@ enum vmcs_field {
    3.47      HOST_RIP                        = 0x00006c16,
    3.48  };
    3.49  
    3.50 +#define VMCS_VPID_WIDTH 16
    3.51 +
    3.52  void vmx_disable_intercept_for_msr(struct vcpu *v, u32 msr);
    3.53  int vmx_read_guest_msr(struct vcpu *v, u32 msr, u64 *val);
    3.54  int vmx_write_guest_msr(struct vcpu *v, u32 msr, u64 val);
     4.1 --- a/xen/include/asm-x86/hvm/vmx/vmx.h	Wed Apr 09 14:10:32 2008 +0100
     4.2 +++ b/xen/include/asm-x86/hvm/vmx/vmx.h	Wed Apr 09 14:34:49 2008 +0100
     4.3 @@ -164,6 +164,7 @@ void vmx_realmode(struct cpu_user_regs *
     4.4  #define VMRESUME_OPCODE ".byte 0x0f,0x01,0xc3\n"
     4.5  #define VMWRITE_OPCODE  ".byte 0x0f,0x79\n"
     4.6  #define INVEPT_OPCODE   ".byte 0x66,0x0f,0x38,0x80\n"   /* m128,r64/32 */
     4.7 +#define INVVPID_OPCODE  ".byte 0x66,0x0f,0x38,0x81\n"   /* m128,r64/32 */
     4.8  #define VMXOFF_OPCODE   ".byte 0x0f,0x01,0xc4\n"
     4.9  #define VMXON_OPCODE    ".byte 0xf3,0x0f,0xc7\n"
    4.10  
    4.11 @@ -260,13 +261,30 @@ static inline void __invept(int ext, u64
    4.12          u64 eptp, gpa;
    4.13      } operand = {eptp, gpa};
    4.14  
    4.15 -    __asm__ __volatile__ ( INVEPT_OPCODE
    4.16 -                           MODRM_EAX_08
    4.17 -                           /* CF==1 or ZF==1 --> rc = -1 */
    4.18 -                           "ja 1f ; ud2 ; 1:\n"
    4.19 -                           :
    4.20 -                           : "a" (&operand), "c" (ext)
    4.21 -                           : "memory");
    4.22 +    asm volatile ( INVEPT_OPCODE
    4.23 +                   MODRM_EAX_08
    4.24 +                   /* CF==1 or ZF==1 --> rc = -1 */
    4.25 +                   "ja 1f ; ud2 ; 1:\n"
    4.26 +                   :
    4.27 +                   : "a" (&operand), "c" (ext)
    4.28 +                   : "memory" );
    4.29 +}
    4.30 +
    4.31 +static inline void __invvpid(int ext, u16 vpid, u64 gva)
    4.32 +{
    4.33 +    struct {
    4.34 +        u64 vpid:16;
    4.35 +        u64 rsvd:48;
    4.36 +        u64 gva;
    4.37 +    } __attribute__ ((packed)) operand = {vpid, 0, gva};
    4.38 +
    4.39 +    asm volatile ( INVVPID_OPCODE
    4.40 +                   MODRM_EAX_08
    4.41 +                   /* CF==1 or ZF==1 --> rc = -1 */
    4.42 +                   "ja 1f ; ud2 ; 1:\n"
    4.43 +                   :
    4.44 +                   : "a" (&operand), "c" (ext)
    4.45 +                   : "memory" );
    4.46  }
    4.47  
    4.48  static inline void ept_sync_all(void)
    4.49 @@ -279,6 +297,18 @@ static inline void ept_sync_all(void)
    4.50  
    4.51  void ept_sync_domain(struct domain *d);
    4.52  
    4.53 +static inline void vpid_sync_vcpu_all(struct vcpu *v)
    4.54 +{
    4.55 +    if ( cpu_has_vmx_vpid )
    4.56 +        __invvpid(1, v->arch.hvm_vmx.vpid, 0);
    4.57 +}
    4.58 +
    4.59 +static inline void vpid_sync_all(void)
    4.60 +{
    4.61 +    if ( cpu_has_vmx_vpid )
    4.62 +        __invvpid(2, 0, 0);
    4.63 +}
    4.64 +
    4.65  static inline void __vmxoff(void)
    4.66  {
    4.67      asm volatile (