ia64/xen-unstable

changeset 10356:d7543cff88ae

[HVM][VMX] Cleanups and fixes to VMCS lifecycle.
1. Maintain a 'launched' software flag to select between
VMLAUNCH and VMRESUME.
2. Take more care with VMPTRLD/VMCLEAR.
Also various other fixes (e.g., safe testing of
condition codes after executing a VMX instruction).

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Sun Jun 11 14:33:16 2006 +0100 (2006-06-11)
parents be05097d5d69
children aced0ee216aa
files xen/arch/ia64/vmx/vmx_init.c xen/arch/x86/hvm/vmx/vmcs.c xen/arch/x86/hvm/vmx/vmx.c xen/arch/x86/hvm/vmx/x86_32/exits.S xen/arch/x86/hvm/vmx/x86_64/exits.S xen/arch/x86/x86_32/asm-offsets.c xen/arch/x86/x86_64/asm-offsets.c xen/include/asm-ia64/vmx_vpd.h xen/include/asm-x86/hvm/vmx/vmcs.h xen/include/asm-x86/hvm/vmx/vmx.h
line diff
     1.1 --- a/xen/arch/ia64/vmx/vmx_init.c	Sun Jun 11 09:54:35 2006 +0100
     1.2 +++ b/xen/arch/ia64/vmx/vmx_init.c	Sun Jun 11 14:33:16 2006 +0100
     1.3 @@ -288,9 +288,6 @@ vmx_final_setup_guest(struct vcpu *v)
     1.4  	/* v->arch.schedule_tail = arch_vmx_do_launch; */
     1.5  	vmx_create_vp(v);
     1.6  
     1.7 -	/* Set this ed to be vmx */
     1.8 -	set_bit(ARCH_VMX_VMCS_LOADED, &v->arch.arch_vmx.flags);
     1.9 -
    1.10  	/* Physical mode emulation initialization, including
    1.11  	* emulation ID allcation and related memory request
    1.12  	*/
     2.1 --- a/xen/arch/x86/hvm/vmx/vmcs.c	Sun Jun 11 09:54:35 2006 +0100
     2.2 +++ b/xen/arch/x86/hvm/vmx/vmcs.c	Sun Jun 11 14:33:16 2006 +0100
     2.3 @@ -42,7 +42,7 @@
     2.4  
     2.5  int vmcs_size;
     2.6  
     2.7 -struct vmcs_struct *alloc_vmcs(void)
     2.8 +struct vmcs_struct *vmx_alloc_vmcs(void)
     2.9  {
    2.10      struct vmcs_struct *vmcs;
    2.11      u32 vmx_msr_low, vmx_msr_high;
    2.12 @@ -64,47 +64,63 @@ static void free_vmcs(struct vmcs_struct
    2.13      free_xenheap_pages(vmcs, order);
    2.14  }
    2.15  
    2.16 -static int load_vmcs(struct arch_vmx_struct *arch_vmx, u64 phys_ptr)
    2.17 +static void __vmx_clear_vmcs(void *info)
    2.18  {
    2.19 -    int error;
    2.20 -
    2.21 -    if ((error = __vmptrld(phys_ptr))) {
    2.22 -        clear_bit(ARCH_VMX_VMCS_LOADED, &arch_vmx->flags);
    2.23 -        return error;
    2.24 -    }
    2.25 -    set_bit(ARCH_VMX_VMCS_LOADED, &arch_vmx->flags);
    2.26 -    return 0;
    2.27 +    struct vcpu *v = info;
    2.28 +    __vmpclear(virt_to_maddr(v->arch.hvm_vmx.vmcs));
    2.29 +    v->arch.hvm_vmx.active_cpu = -1;
    2.30 +    v->arch.hvm_vmx.launched   = 0;
    2.31  }
    2.32  
    2.33 -static void vmx_smp_clear_vmcs(void *info)
    2.34 +static void vmx_clear_vmcs(struct vcpu *v)
    2.35  {
    2.36 -    struct vcpu *v = (struct vcpu *)info;
    2.37 +    unsigned int cpu = v->arch.hvm_vmx.active_cpu;
    2.38  
    2.39 -    ASSERT(hvm_guest(v));
    2.40 -
    2.41 -    if (v->arch.hvm_vmx.launch_cpu == smp_processor_id())
    2.42 -        __vmpclear(virt_to_maddr(v->arch.hvm_vmx.vmcs));
    2.43 +    if ( (cpu == -1) || (cpu == smp_processor_id()) )
    2.44 +        __vmx_clear_vmcs(v);
    2.45 +    else
    2.46 +        on_selected_cpus(cpumask_of_cpu(cpu), __vmx_clear_vmcs, v, 1, 1);
    2.47  }
    2.48  
    2.49 -void vmx_request_clear_vmcs(struct vcpu *v)
    2.50 +static void vmx_load_vmcs(struct vcpu *v)
    2.51  {
    2.52 -    ASSERT(hvm_guest(v));
    2.53 -
    2.54 -    if (v->arch.hvm_vmx.launch_cpu == smp_processor_id())
    2.55 -        __vmpclear(virt_to_maddr(v->arch.hvm_vmx.vmcs));
    2.56 -    else
    2.57 -        smp_call_function(vmx_smp_clear_vmcs, v, 1, 1);
    2.58 +    __vmptrld(virt_to_maddr(v->arch.hvm_vmx.vmcs));
    2.59 +    v->arch.hvm_vmx.active_cpu = smp_processor_id();
    2.60  }
    2.61  
    2.62 -#if 0
    2.63 -static int store_vmcs(struct arch_vmx_struct *arch_vmx, u64 phys_ptr)
    2.64 +void vmx_vmcs_enter(struct vcpu *v)
    2.65  {
    2.66 -    /* take the current VMCS */
    2.67 -    __vmptrst(phys_ptr);
    2.68 -    clear_bit(ARCH_VMX_VMCS_LOADED, &arch_vmx->flags);
    2.69 -    return 0;
    2.70 +    /*
    2.71 +     * NB. We must *always* run an HVM VCPU on its own VMCS, except for
    2.72 +     * vmx_vmcs_enter/exit critical regions. This leads to some XXX TODOs XXX:
    2.73 +     *  1. Move construct_vmcs() much earlier, to domain creation or
    2.74 +     *     context initialisation.
    2.75 +     *  2. VMPTRLD as soon as we context-switch to a HVM VCPU.
    2.76 +     *  3. VMCS destruction needs to happen later (from domain_destroy()).
    2.77 +     */
    2.78 +    if ( v == current )
    2.79 +        return;
    2.80 +
    2.81 +    vcpu_pause(v);
    2.82 +    spin_lock(&v->arch.hvm_vmx.vmcs_lock);
    2.83 +
    2.84 +    vmx_clear_vmcs(v);
    2.85 +    vmx_load_vmcs(v);
    2.86  }
    2.87 -#endif
    2.88 +
    2.89 +void vmx_vmcs_exit(struct vcpu *v)
    2.90 +{
    2.91 +    if ( v == current )
    2.92 +        return;
    2.93 +
    2.94 +    /* Don't confuse arch_vmx_do_resume (for @v or @current!) */
    2.95 +    vmx_clear_vmcs(v);
    2.96 +    if ( hvm_guest(current) )
    2.97 +        vmx_load_vmcs(current);
    2.98 +
    2.99 +    spin_unlock(&v->arch.hvm_vmx.vmcs_lock);
   2.100 +    vcpu_unpause(v);
   2.101 +}
   2.102  
   2.103  static inline int construct_vmcs_controls(struct arch_vmx_struct *arch_vmx)
   2.104  {
   2.105 @@ -247,7 +263,6 @@ static void vmx_do_launch(struct vcpu *v
   2.106      __vmwrite(HOST_CR3, pagetable_get_paddr(v->arch.monitor_table));
   2.107  
   2.108      v->arch.schedule_tail = arch_vmx_do_resume;
   2.109 -    v->arch.hvm_vmx.launch_cpu = smp_processor_id();
   2.110  
   2.111      /* init guest tsc to start from 0 */
   2.112      set_guest_time(v, 0);
   2.113 @@ -410,53 +425,49 @@ static inline int construct_vmcs_host(vo
   2.114  /*
   2.115   * Need to extend to support full virtualization.
   2.116   */
   2.117 -static int construct_vmcs(struct arch_vmx_struct *arch_vmx,
   2.118 +static int construct_vmcs(struct vcpu *v,
   2.119                            cpu_user_regs_t *regs)
   2.120  {
   2.121 +    struct arch_vmx_struct *arch_vmx = &v->arch.hvm_vmx;
   2.122      int error;
   2.123      long rc;
   2.124 -    u64 vmcs_phys_ptr;
   2.125  
   2.126      memset(arch_vmx, 0, sizeof(struct arch_vmx_struct));
   2.127  
   2.128 +    spin_lock_init(&arch_vmx->vmcs_lock);
   2.129 +    arch_vmx->active_cpu = -1;
   2.130 +
   2.131      /*
   2.132       * Create a new VMCS
   2.133       */
   2.134 -    if (!(arch_vmx->vmcs = alloc_vmcs())) {
   2.135 +    if (!(arch_vmx->vmcs = vmx_alloc_vmcs())) {
   2.136          printk("Failed to create a new VMCS\n");
   2.137 -        rc = -ENOMEM;
   2.138 -        goto err_out;
   2.139 +        return -ENOMEM;
   2.140      }
   2.141 -    vmcs_phys_ptr = (u64) virt_to_maddr(arch_vmx->vmcs);
   2.142  
   2.143 -    if ((error = __vmpclear(vmcs_phys_ptr))) {
   2.144 -        printk("construct_vmcs: VMCLEAR failed\n");
   2.145 -        rc = -EINVAL;
   2.146 -        goto err_out;
   2.147 -    }
   2.148 -    if ((error = load_vmcs(arch_vmx, vmcs_phys_ptr))) {
   2.149 -        printk("construct_vmcs: load_vmcs failed: VMCS = %lx\n",
   2.150 -               (unsigned long) vmcs_phys_ptr);
   2.151 -        rc = -EINVAL;
   2.152 -        goto err_out;
   2.153 -    }
   2.154 +    vmx_clear_vmcs(v);
   2.155 +    vmx_load_vmcs(v);
   2.156 +
   2.157      if ((error = construct_vmcs_controls(arch_vmx))) {
   2.158          printk("construct_vmcs: construct_vmcs_controls failed\n");
   2.159          rc = -EINVAL;
   2.160          goto err_out;
   2.161      }
   2.162 +
   2.163      /* host selectors */
   2.164      if ((error = construct_vmcs_host())) {
   2.165          printk("construct_vmcs: construct_vmcs_host failed\n");
   2.166          rc = -EINVAL;
   2.167          goto err_out;
   2.168      }
   2.169 +
   2.170      /* guest selectors */
   2.171      if ((error = construct_init_vmcs_guest(regs))) {
   2.172          printk("construct_vmcs: construct_vmcs_guest failed\n");
   2.173          rc = -EINVAL;
   2.174          goto err_out;
   2.175      }
   2.176 +
   2.177      if ((error |= __vmwrite(EXCEPTION_BITMAP,
   2.178                              MONITOR_DEFAULT_EXCEPTION_BITMAP))) {
   2.179          printk("construct_vmcs: setting Exception bitmap failed\n");
   2.180 @@ -472,12 +483,16 @@ static int construct_vmcs(struct arch_vm
   2.181      return 0;
   2.182  
   2.183  err_out:
   2.184 -    destroy_vmcs(arch_vmx);
   2.185 +    vmx_destroy_vmcs(v);
   2.186      return rc;
   2.187  }
   2.188  
   2.189 -void destroy_vmcs(struct arch_vmx_struct *arch_vmx)
   2.190 +void vmx_destroy_vmcs(struct vcpu *v)
   2.191  {
   2.192 +    struct arch_vmx_struct *arch_vmx = &v->arch.hvm_vmx;
   2.193 +
   2.194 +    vmx_clear_vmcs(v);
   2.195 +
   2.196      free_vmcs(arch_vmx->vmcs);
   2.197      arch_vmx->vmcs = NULL;
   2.198  
   2.199 @@ -506,22 +521,20 @@ void vm_resume_fail(unsigned long eflags
   2.200  
   2.201  void arch_vmx_do_resume(struct vcpu *v)
   2.202  {
   2.203 -    if ( v->arch.hvm_vmx.launch_cpu == smp_processor_id() )
   2.204 +    if ( v->arch.hvm_vmx.active_cpu == smp_processor_id() )
   2.205      {
   2.206 -        load_vmcs(&v->arch.hvm_vmx, virt_to_maddr(v->arch.hvm_vmx.vmcs));
   2.207 -        vmx_do_resume(v);
   2.208 -        reset_stack_and_jump(vmx_asm_do_resume);
   2.209 +        vmx_load_vmcs(v);
   2.210      }
   2.211      else
   2.212      {
   2.213 -        vmx_request_clear_vmcs(v);
   2.214 -        load_vmcs(&v->arch.hvm_vmx, virt_to_maddr(v->arch.hvm_vmx.vmcs));
   2.215 +        vmx_clear_vmcs(v);
   2.216 +        vmx_load_vmcs(v);
   2.217          vmx_migrate_timers(v);
   2.218          vmx_set_host_env(v);
   2.219 -        vmx_do_resume(v);
   2.220 -        v->arch.hvm_vmx.launch_cpu = smp_processor_id();
   2.221 -        reset_stack_and_jump(vmx_asm_do_relaunch);
   2.222      }
   2.223 +
   2.224 +    vmx_do_resume(v);
   2.225 +    reset_stack_and_jump(vmx_asm_do_vmentry);
   2.226  }
   2.227  
   2.228  void arch_vmx_do_launch(struct vcpu *v)
   2.229 @@ -529,7 +542,7 @@ void arch_vmx_do_launch(struct vcpu *v)
   2.230      int error;
   2.231      cpu_user_regs_t *regs = &current->arch.guest_context.user_regs;
   2.232  
   2.233 -    error = construct_vmcs(&v->arch.hvm_vmx, regs);
   2.234 +    error = construct_vmcs(v, regs);
   2.235      if ( error < 0 )
   2.236      {
   2.237          if (v->vcpu_id == 0) {
   2.238 @@ -540,7 +553,7 @@ void arch_vmx_do_launch(struct vcpu *v)
   2.239          domain_crash_synchronous();
   2.240      }
   2.241      vmx_do_launch(v);
   2.242 -    reset_stack_and_jump(vmx_asm_do_launch);
   2.243 +    reset_stack_and_jump(vmx_asm_do_vmentry);
   2.244  }
   2.245  
   2.246  
   2.247 @@ -613,17 +626,9 @@ static void vmcs_dump(unsigned char ch)
   2.248              }
   2.249              printk("\tVCPU %d\n", v->vcpu_id);
   2.250  
   2.251 -            if (v != current) {
   2.252 -                vcpu_pause(v);
   2.253 -                __vmptrld(virt_to_maddr(v->arch.hvm_vmx.vmcs));
   2.254 -            }
   2.255 -
   2.256 +            vmx_vmcs_enter(v);
   2.257              vmcs_dump_vcpu();
   2.258 -
   2.259 -            if (v != current) {
   2.260 -                __vmptrld(virt_to_maddr(current->arch.hvm_vmx.vmcs));
   2.261 -                vcpu_unpause(v);
   2.262 -            }
   2.263 +            vmx_vmcs_exit(v);
   2.264          }
   2.265      }
   2.266  
     3.1 --- a/xen/arch/x86/hvm/vmx/vmx.c	Sun Jun 11 09:54:35 2006 +0100
     3.2 +++ b/xen/arch/x86/hvm/vmx/vmx.c	Sun Jun 11 14:33:16 2006 +0100
     3.3 @@ -91,8 +91,7 @@ static void vmx_relinquish_guest_resourc
     3.4      {
     3.5          if ( !test_bit(_VCPUF_initialised, &v->vcpu_flags) )
     3.6              continue;
     3.7 -        vmx_request_clear_vmcs(v);
     3.8 -        destroy_vmcs(&v->arch.hvm_vmx);
     3.9 +        vmx_destroy_vmcs(v);
    3.10          free_monitor_pagetable(v);
    3.11          kill_timer(&v->arch.hvm_vmx.hlt_timer);
    3.12          if ( hvm_apic_support(v->domain) && (VLAPIC(v) != NULL) )
    3.13 @@ -402,54 +401,10 @@ void vmx_migrate_timers(struct vcpu *v)
    3.14          migrate_timer(&(VLAPIC(v)->vlapic_timer), v->processor);
    3.15  }
    3.16  
    3.17 -struct vmx_cpu_guest_regs_callback_info {
    3.18 -    struct vcpu *v;
    3.19 -    struct cpu_user_regs *regs;
    3.20 -    unsigned long *crs;
    3.21 -};
    3.22 -
    3.23 -static void vmx_store_cpu_guest_regs(
    3.24 -    struct vcpu *v, struct cpu_user_regs *regs, unsigned long *crs);
    3.25 -
    3.26 -static void vmx_load_cpu_guest_regs(
    3.27 -    struct vcpu *v, struct cpu_user_regs *regs);
    3.28 -
    3.29 -static void vmx_store_cpu_guest_regs_callback(void *data)
    3.30 -{
    3.31 -    struct vmx_cpu_guest_regs_callback_info *info = data;
    3.32 -    vmx_store_cpu_guest_regs(info->v, info->regs, info->crs);
    3.33 -}
    3.34 -
    3.35 -static void vmx_load_cpu_guest_regs_callback(void *data)
    3.36 -{
    3.37 -    struct vmx_cpu_guest_regs_callback_info *info = data;
    3.38 -    vmx_load_cpu_guest_regs(info->v, info->regs);
    3.39 -}
    3.40 -
    3.41  static void vmx_store_cpu_guest_regs(
    3.42      struct vcpu *v, struct cpu_user_regs *regs, unsigned long *crs)
    3.43  {
    3.44 -    if ( v != current )
    3.45 -    {
    3.46 -        /* Non-current VCPUs must be paused to get a register snapshot. */
    3.47 -        ASSERT(atomic_read(&v->pausecnt) != 0);
    3.48 -
    3.49 -        if ( v->arch.hvm_vmx.launch_cpu != smp_processor_id() )
    3.50 -        {
    3.51 -            /* Get register details from remote CPU. */
    3.52 -            struct vmx_cpu_guest_regs_callback_info info = {
    3.53 -                .v = v, .regs = regs, .crs = crs };
    3.54 -            cpumask_t cpumask = cpumask_of_cpu(v->arch.hvm_vmx.launch_cpu);
    3.55 -            on_selected_cpus(cpumask, vmx_store_cpu_guest_regs_callback,
    3.56 -                             &info, 1, 1);
    3.57 -            return;
    3.58 -        }
    3.59 -
    3.60 -        /* Register details are on this CPU. Load the correct VMCS. */
    3.61 -        __vmptrld(virt_to_maddr(v->arch.hvm_vmx.vmcs));
    3.62 -    }
    3.63 -
    3.64 -    ASSERT(v->arch.hvm_vmx.launch_cpu == smp_processor_id());
    3.65 +    vmx_vmcs_enter(v);
    3.66  
    3.67      if ( regs != NULL )
    3.68      {
    3.69 @@ -471,9 +426,7 @@ static void vmx_store_cpu_guest_regs(
    3.70          __vmread(CR4_READ_SHADOW, &crs[4]);
    3.71      }
    3.72  
    3.73 -    /* Reload current VCPU's VMCS if it was temporarily unloaded. */
    3.74 -    if ( (v != current) && hvm_guest(current) )
    3.75 -        __vmptrld(virt_to_maddr(current->arch.hvm_vmx.vmcs));
    3.76 +    vmx_vmcs_exit(v);
    3.77  }
    3.78  
    3.79  /*
    3.80 @@ -517,26 +470,7 @@ static void fixup_vm86_seg_bases(struct 
    3.81  
    3.82  void vmx_load_cpu_guest_regs(struct vcpu *v, struct cpu_user_regs *regs)
    3.83  {
    3.84 -    if ( v != current )
    3.85 -    {
    3.86 -        /* Non-current VCPUs must be paused to set the register snapshot. */
    3.87 -        ASSERT(atomic_read(&v->pausecnt) != 0);
    3.88 -
    3.89 -        if ( v->arch.hvm_vmx.launch_cpu != smp_processor_id() )
    3.90 -        {
    3.91 -            struct vmx_cpu_guest_regs_callback_info info = {
    3.92 -                .v = v, .regs = regs };
    3.93 -            cpumask_t cpumask = cpumask_of_cpu(v->arch.hvm_vmx.launch_cpu);
    3.94 -            on_selected_cpus(cpumask, vmx_load_cpu_guest_regs_callback,
    3.95 -                             &info, 1, 1);
    3.96 -            return;
    3.97 -        }
    3.98 -
    3.99 -        /* Register details are on this CPU. Load the correct VMCS. */
   3.100 -        __vmptrld(virt_to_maddr(v->arch.hvm_vmx.vmcs));
   3.101 -    }
   3.102 -
   3.103 -    ASSERT(v->arch.hvm_vmx.launch_cpu == smp_processor_id());
   3.104 +    vmx_vmcs_enter(v);
   3.105  
   3.106      __vmwrite(GUEST_SS_SELECTOR, regs->ss);
   3.107      __vmwrite(GUEST_DS_SELECTOR, regs->ds);
   3.108 @@ -557,9 +491,7 @@ void vmx_load_cpu_guest_regs(struct vcpu
   3.109      __vmwrite(GUEST_CS_SELECTOR, regs->cs);
   3.110      __vmwrite(GUEST_RIP, regs->eip);
   3.111  
   3.112 -    /* Reload current VCPU's VMCS if it was temporarily unloaded. */
   3.113 -    if ( (v != current) && hvm_guest(current) )
   3.114 -        __vmptrld(virt_to_maddr(current->arch.hvm_vmx.vmcs));
   3.115 +    vmx_vmcs_exit(v);
   3.116  }
   3.117  
   3.118  int vmx_realmode(struct vcpu *v)
   3.119 @@ -688,17 +620,20 @@ int start_vmx(void)
   3.120  
   3.121      set_in_cr4(X86_CR4_VMXE);   /* Enable VMXE */
   3.122  
   3.123 -    if (!(vmcs = alloc_vmcs())) {
   3.124 +    if (!(vmcs = vmx_alloc_vmcs())) {
   3.125          printk("Failed to allocate VMCS\n");
   3.126          return 0;
   3.127      }
   3.128  
   3.129      phys_vmcs = (u64) virt_to_maddr(vmcs);
   3.130  
   3.131 -    if (!(__vmxon(phys_vmcs))) {
   3.132 -        printk("VMXON is done\n");
   3.133 +    if (__vmxon(phys_vmcs)) {
   3.134 +        printk("VMXON failed\n");
   3.135 +        return 0;
   3.136      }
   3.137  
   3.138 +    printk("VMXON is done\n");
   3.139 +
   3.140      vmx_save_init_msrs();
   3.141  
   3.142      /* Setup HVM interfaces */
     4.1 --- a/xen/arch/x86/hvm/vmx/x86_32/exits.S	Sun Jun 11 09:54:35 2006 +0100
     4.2 +++ b/xen/arch/x86/hvm/vmx/x86_32/exits.S	Sun Jun 11 14:33:16 2006 +0100
     4.3 @@ -78,69 +78,49 @@
     4.4          addl $(NR_SKIPPED_REGS*4), %esp
     4.5  
     4.6          ALIGN
     4.7 -
     4.8  ENTRY(vmx_asm_vmexit_handler)
     4.9          /* selectors are restored/saved by VMX */
    4.10          HVM_SAVE_ALL_NOSEGREGS
    4.11          call vmx_trace_vmexit
    4.12          call vmx_vmexit_handler
    4.13 -        jmp vmx_asm_do_resume
    4.14 +        jmp vmx_asm_do_vmentry
    4.15  
    4.16 -.macro vmx_asm_common launch, initialized
    4.17 -1:
    4.18 -/* vmx_test_all_events */
    4.19 -        .if \initialized
    4.20 +        ALIGN
    4.21 +vmx_process_softirqs:
    4.22 +        sti       
    4.23 +        call do_softirq
    4.24 +        jmp vmx_asm_do_vmentry
    4.25 +
    4.26 +        ALIGN
    4.27 +ENTRY(vmx_asm_do_vmentry)
    4.28          GET_CURRENT(%ebx)
    4.29 -/*test_all_events:*/
    4.30 -        xorl %ecx,%ecx
    4.31 -        notl %ecx
    4.32          cli                             # tests must not race interrupts
    4.33 -/*test_softirqs:*/  
    4.34 +
    4.35          movl VCPU_processor(%ebx),%eax
    4.36          shl  $IRQSTAT_shift,%eax
    4.37 -        test %ecx,irq_stat(%eax,1)
    4.38 -        jnz 2f
    4.39 +        cmpl $0,irq_stat(%eax,1)
    4.40 +        jnz  vmx_process_softirqs
    4.41  
    4.42 -/* vmx_restore_all_guest */
    4.43          call vmx_intr_assist
    4.44          call vmx_load_cr2
    4.45          call vmx_trace_vmentry
    4.46 -        .endif
    4.47 +
    4.48 +        cmpl $0,VCPU_vmx_launched(%ebx)
    4.49 +        je   vmx_launch
    4.50 +
    4.51 +/*vmx_resume:*/
    4.52          HVM_RESTORE_ALL_NOSEGREGS
    4.53 -        /* 
    4.54 -         * Check if we are going back to VMX-based VM
    4.55 -         * By this time, all the setups in the VMCS must be complete.
    4.56 -         */
    4.57 -        .if \launch
    4.58 +        /* VMRESUME */
    4.59 +        .byte 0x0f,0x01,0xc3
    4.60 +        pushf
    4.61 +        call vm_resume_fail
    4.62 +        ud2
    4.63 +
    4.64 +vmx_launch:
    4.65 +        movl $1,VCPU_vmx_launched(%ebx)
    4.66 +        HVM_RESTORE_ALL_NOSEGREGS
    4.67          /* VMLAUNCH */
    4.68          .byte 0x0f,0x01,0xc2
    4.69          pushf
    4.70          call vm_launch_fail
    4.71 -        .else
    4.72 -        /* VMRESUME */
    4.73 -        .byte 0x0f,0x01,0xc3
    4.74 -        pushf
    4.75 -        call vm_resume_fail
    4.76 -        .endif
    4.77 -        /* Should never reach here */
    4.78 -        hlt
    4.79 -
    4.80 -        ALIGN
    4.81 -        .if \initialized
    4.82 -2:
    4.83 -/* vmx_process_softirqs */
    4.84 -        sti       
    4.85 -        call do_softirq
    4.86 -        jmp 1b
    4.87 -        ALIGN
    4.88 -        .endif
    4.89 -.endm
    4.90 -
    4.91 -ENTRY(vmx_asm_do_launch)
    4.92 -    vmx_asm_common 1, 0
    4.93 -
    4.94 -ENTRY(vmx_asm_do_resume)
    4.95 -    vmx_asm_common 0, 1
    4.96 -
    4.97 -ENTRY(vmx_asm_do_relaunch)
    4.98 -    vmx_asm_common 1, 1
    4.99 +        ud2
     5.1 --- a/xen/arch/x86/hvm/vmx/x86_64/exits.S	Sun Jun 11 09:54:35 2006 +0100
     5.2 +++ b/xen/arch/x86/hvm/vmx/x86_64/exits.S	Sun Jun 11 14:33:16 2006 +0100
     5.3 @@ -88,68 +88,51 @@
     5.4          popq %rdi;                              \
     5.5          addq $(NR_SKIPPED_REGS*8), %rsp;
     5.6  
     5.7 +        ALIGN
     5.8  ENTRY(vmx_asm_vmexit_handler)
     5.9          /* selectors are restored/saved by VMX */
    5.10          HVM_SAVE_ALL_NOSEGREGS
    5.11          call vmx_trace_vmexit
    5.12          call vmx_vmexit_handler
    5.13 -        jmp vmx_asm_do_resume
    5.14 +        jmp vmx_asm_do_vmentry
    5.15  
    5.16 -.macro vmx_asm_common launch, initialized 
    5.17 -1:
    5.18 -        .if \initialized
    5.19 -/* vmx_test_all_events */
    5.20 +        ALIGN
    5.21 +vmx_process_softirqs:
    5.22 +        sti       
    5.23 +        call do_softirq
    5.24 +        jmp vmx_asm_do_vmentry
    5.25 +
    5.26 +        ALIGN
    5.27 +ENTRY(vmx_asm_do_vmentry)
    5.28          GET_CURRENT(%rbx)
    5.29 -/* test_all_events: */
    5.30          cli                             # tests must not race interrupts
    5.31 -/*test_softirqs:*/  
    5.32 +
    5.33          movl  VCPU_processor(%rbx),%eax
    5.34          shl   $IRQSTAT_shift,%rax
    5.35 -        leaq  irq_stat(%rip), %rdx
    5.36 -        testl $~0,(%rdx,%rax,1)
    5.37 -        jnz  2f 
    5.38 +        leaq  irq_stat(%rip),%rdx
    5.39 +        cmpl  $0,(%rdx,%rax,1)
    5.40 +        jnz   vmx_process_softirqs
    5.41  
    5.42 -/* vmx_restore_all_guest */
    5.43          call vmx_intr_assist
    5.44          call vmx_load_cr2
    5.45          call vmx_trace_vmentry
    5.46 -        .endif
    5.47 -        /* 
    5.48 -         * Check if we are going back to VMX-based VM
    5.49 -         * By this time, all the setups in the VMCS must be complete.
    5.50 -         */
    5.51 +
    5.52 +        cmpl $0,VCPU_vmx_launched(%rbx)
    5.53 +        je   vmx_launch
    5.54 +
    5.55 +/*vmx_resume:*/
    5.56          HVM_RESTORE_ALL_NOSEGREGS
    5.57 -        .if \launch
    5.58 +        /* VMRESUME */
    5.59 +        .byte 0x0f,0x01,0xc3
    5.60 +        pushfq
    5.61 +        call vm_resume_fail
    5.62 +        ud2
    5.63 +
    5.64 +vmx_launch:
    5.65 +        movl $1,VCPU_vmx_launched(%rbx)
    5.66 +        HVM_RESTORE_ALL_NOSEGREGS
    5.67          /* VMLAUNCH */
    5.68          .byte 0x0f,0x01,0xc2
    5.69          pushfq
    5.70          call vm_launch_fail
    5.71 -        .else
    5.72 -        /* VMRESUME */
    5.73 -        .byte 0x0f,0x01,0xc3
    5.74 -        pushfq
    5.75 -        call vm_resume_fail
    5.76 -        .endif
    5.77 -        /* Should never reach here */
    5.78 -        hlt
    5.79 -
    5.80 -        ALIGN
    5.81 -
    5.82 -        .if \initialized
    5.83 -2:
    5.84 -/* vmx_process_softirqs */
    5.85 -        sti       
    5.86 -        call do_softirq
    5.87 -        jmp 1b
    5.88 -        ALIGN
    5.89 -        .endif
    5.90 -.endm
    5.91 -
    5.92 -ENTRY(vmx_asm_do_launch)
    5.93 -      vmx_asm_common 1, 0
    5.94 -
    5.95 -ENTRY(vmx_asm_do_resume)
    5.96 -      vmx_asm_common 0, 1
    5.97 -
    5.98 -ENTRY(vmx_asm_do_relaunch)
    5.99 -      vmx_asm_common 1, 1
   5.100 +        ud2
     6.1 --- a/xen/arch/x86/x86_32/asm-offsets.c	Sun Jun 11 09:54:35 2006 +0100
     6.2 +++ b/xen/arch/x86/x86_32/asm-offsets.c	Sun Jun 11 14:33:16 2006 +0100
     6.3 @@ -86,6 +86,9 @@ void __dummy__(void)
     6.4      OFFSET(VCPU_svm_vmexit_tsc, struct vcpu, arch.hvm_svm.vmexit_tsc);
     6.5      BLANK();
     6.6  
     6.7 +    OFFSET(VCPU_vmx_launched, struct vcpu, arch.hvm_vmx.launched);
     6.8 +    BLANK();
     6.9 +
    6.10      OFFSET(VMCB_rax, struct vmcb_struct, rax);
    6.11      OFFSET(VMCB_tsc_offset, struct vmcb_struct, tsc_offset);
    6.12      BLANK();
     7.1 --- a/xen/arch/x86/x86_64/asm-offsets.c	Sun Jun 11 09:54:35 2006 +0100
     7.2 +++ b/xen/arch/x86/x86_64/asm-offsets.c	Sun Jun 11 14:33:16 2006 +0100
     7.3 @@ -80,6 +80,9 @@ void __dummy__(void)
     7.4      OFFSET(VCPU_svm_vmexit_tsc, struct vcpu, arch.hvm_svm.vmexit_tsc);
     7.5      BLANK();
     7.6  
     7.7 +    OFFSET(VCPU_vmx_launched, struct vcpu, arch.hvm_vmx.launched);
     7.8 +    BLANK();
     7.9 +
    7.10      OFFSET(VMCB_rax, struct vmcb_struct, rax);
    7.11      OFFSET(VMCB_tsc_offset, struct vmcb_struct, tsc_offset);
    7.12      BLANK();
     8.1 --- a/xen/include/asm-ia64/vmx_vpd.h	Sun Jun 11 09:54:35 2006 +0100
     8.2 +++ b/xen/include/asm-ia64/vmx_vpd.h	Sun Jun 11 14:33:16 2006 +0100
     8.3 @@ -104,9 +104,6 @@ struct arch_vmx_struct {
     8.4  
     8.5  #define VMX_DOMAIN(d)   d->arch.arch_vmx.flags
     8.6  
     8.7 -#define ARCH_VMX_VMCS_LOADED    0       /* VMCS has been loaded and active */
     8.8 -#define ARCH_VMX_VMCS_LAUNCH    1       /* Needs VMCS launch */
     8.9 -#define ARCH_VMX_VMCS_RESUME    2       /* Needs VMCS resume */
    8.10  #define ARCH_VMX_IO_WAIT        3       /* Waiting for I/O completion */
    8.11  #define ARCH_VMX_INTR_ASSIST    4       /* Need DM's assist to issue intr */
    8.12  #define ARCH_VMX_CONTIG_MEM 	5	/* Need contiguous machine pages */
     9.1 --- a/xen/include/asm-x86/hvm/vmx/vmcs.h	Sun Jun 11 09:54:35 2006 +0100
     9.2 +++ b/xen/include/asm-x86/hvm/vmx/vmcs.h	Sun Jun 11 14:33:16 2006 +0100
     9.3 @@ -65,34 +65,46 @@ struct vmx_msr_state {
     9.4  };
     9.5  
     9.6  struct arch_vmx_struct {
     9.7 -    struct vmcs_struct      *vmcs;  /* VMCS pointer in virtual. */
     9.8 -    unsigned int            launch_cpu; /* VMCS is valid on this CPU. */
     9.9 -    u32                     exec_control; /* cache of cpu execution control */
    9.10 -    u32                     vector_injected; /* if there is vector installed in the INTR_INFO_FIELD */
    9.11 -    unsigned long           flags;  /* VMCS flags */
    9.12 -    unsigned long           cpu_cr0; /* copy of guest CR0 */
    9.13 -    unsigned long           cpu_shadow_cr0; /* copy of guest read shadow CR0 */
    9.14 -    unsigned long           cpu_cr2; /* save CR2 */
    9.15 -    unsigned long           cpu_cr3;
    9.16 -    unsigned long           cpu_state;
    9.17 -    unsigned long           cpu_based_exec_control;
    9.18 -    struct vmx_msr_state    msr_content;
    9.19 -    void                    *io_bitmap_a, *io_bitmap_b;
    9.20 -    struct timer            hlt_timer;  /* hlt ins emulation wakeup timer */
    9.21 +    /* Virtual address of VMCS. */
    9.22 +    struct vmcs_struct  *vmcs;
    9.23 +
    9.24 +    /* Protects remote usage of VMCS (VMPTRLD/VMCLEAR). */
    9.25 +    spinlock_t           vmcs_lock;
    9.26 +
    9.27 +    /*
    9.28 +     * Activation and launch status of this VMCS.
    9.29 +     *  - Activated on a CPU by VMPTRLD. Deactivated by VMCLEAR.
    9.30 +     *  - Launched on active CPU by VMLAUNCH when current VMCS.
    9.31 +     */
    9.32 +    int                  active_cpu;
    9.33 +    int                  launched;
    9.34 +
    9.35 +    /* Cache of cpu execution control. */
    9.36 +    u32                  exec_control;
    9.37 +
    9.38 +    /* If there is vector installed in the INTR_INFO_FIELD. */
    9.39 +    u32                  vector_injected;
    9.40 +
    9.41 +    unsigned long        cpu_cr0; /* copy of guest CR0 */
    9.42 +    unsigned long        cpu_shadow_cr0; /* copy of guest read shadow CR0 */
    9.43 +    unsigned long        cpu_cr2; /* save CR2 */
    9.44 +    unsigned long        cpu_cr3;
    9.45 +    unsigned long        cpu_state;
    9.46 +    unsigned long        cpu_based_exec_control;
    9.47 +    struct vmx_msr_state msr_content;
    9.48 +    void                *io_bitmap_a, *io_bitmap_b;
    9.49 +    struct timer         hlt_timer;  /* hlt ins emulation wakeup timer */
    9.50  };
    9.51  
    9.52  #define vmx_schedule_tail(next)         \
    9.53      (next)->thread.arch_vmx.arch_vmx_schedule_tail((next))
    9.54  
    9.55 -#define ARCH_VMX_VMCS_LOADED    0       /* VMCS has been loaded and active */
    9.56 -#define ARCH_VMX_VMCS_LAUNCH    1       /* Needs VMCS launch */
    9.57 -#define ARCH_VMX_VMCS_RESUME    2       /* Needs VMCS resume */
    9.58 +void vmx_do_resume(struct vcpu *);
    9.59  
    9.60 -void vmx_do_resume(struct vcpu *);
    9.61 -struct vmcs_struct *alloc_vmcs(void);
    9.62 -void destroy_vmcs(struct arch_vmx_struct *arch_vmx);
    9.63 -
    9.64 -extern void vmx_request_clear_vmcs(struct vcpu *v);
    9.65 +struct vmcs_struct *vmx_alloc_vmcs(void);
    9.66 +void vmx_destroy_vmcs(struct vcpu *v);
    9.67 +void vmx_vmcs_enter(struct vcpu *v);
    9.68 +void vmx_vmcs_exit(struct vcpu *v);
    9.69  
    9.70  #define VMCS_USE_HOST_ENV       1
    9.71  #define VMCS_USE_SEPARATE_ENV   0
    10.1 --- a/xen/include/asm-x86/hvm/vmx/vmx.h	Sun Jun 11 09:54:35 2006 +0100
    10.2 +++ b/xen/include/asm-x86/hvm/vmx/vmx.h	Sun Jun 11 14:33:16 2006 +0100
    10.3 @@ -27,8 +27,7 @@
    10.4  #include <asm/i387.h>
    10.5  
    10.6  extern void vmx_asm_vmexit_handler(struct cpu_user_regs);
    10.7 -extern void vmx_asm_do_resume(void);
    10.8 -extern void vmx_asm_do_launch(void);
    10.9 +extern void vmx_asm_do_vmentry(void);
   10.10  extern void vmx_intr_assist(void);
   10.11  extern void vmx_migrate_timers(struct vcpu *v);
   10.12  extern void arch_vmx_do_launch(struct vcpu *);
   10.13 @@ -200,22 +199,18 @@ extern unsigned int cpu_rev;
   10.14  #define MODRM_EAX_07    ".byte 0x38\n" /* [EAX], with reg/opcode: /7 */
   10.15  #define MODRM_EAX_ECX   ".byte 0xc1\n" /* [EAX], [ECX] */
   10.16  
   10.17 -static inline int __vmptrld (u64 addr)
   10.18 +static inline void __vmptrld(u64 addr)
   10.19  {
   10.20 -    unsigned long eflags;
   10.21      __asm__ __volatile__ ( VMPTRLD_OPCODE
   10.22                             MODRM_EAX_06
   10.23 +                           /* CF==1 or ZF==1 --> crash (ud2) */
   10.24 +                           "ja 1f ; ud2 ; 1:\n"
   10.25                             :
   10.26                             : "a" (&addr) 
   10.27                             : "memory");
   10.28 -
   10.29 -    __save_flags(eflags);
   10.30 -    if (eflags & X86_EFLAGS_ZF || eflags & X86_EFLAGS_CF)
   10.31 -        return -1;
   10.32 -    return 0;
   10.33  }
   10.34  
   10.35 -static inline void __vmptrst (u64 addr)
   10.36 +static inline void __vmptrst(u64 addr)
   10.37  {
   10.38      __asm__ __volatile__ ( VMPTRST_OPCODE
   10.39                             MODRM_EAX_07
   10.40 @@ -224,31 +219,30 @@ static inline void __vmptrst (u64 addr)
   10.41                             : "memory");
   10.42  }
   10.43  
   10.44 -static inline int __vmpclear (u64 addr)
   10.45 +static inline void __vmpclear(u64 addr)
   10.46  {
   10.47 -    unsigned long eflags;
   10.48 -
   10.49      __asm__ __volatile__ ( VMCLEAR_OPCODE
   10.50                             MODRM_EAX_06
   10.51 +                           /* CF==1 or ZF==1 --> crash (ud2) */
   10.52 +                           "ja 1f ; ud2 ; 1:\n"
   10.53                             :
   10.54                             : "a" (&addr) 
   10.55                             : "memory");
   10.56 -    __save_flags(eflags);
   10.57 -    if (eflags & X86_EFLAGS_ZF || eflags & X86_EFLAGS_CF)
   10.58 -        return -1;
   10.59 -    return 0;
   10.60  }
   10.61  
   10.62  #define __vmread(x, ptr) ___vmread((x), (ptr), sizeof(*(ptr)))
   10.63  
   10.64 -static always_inline int ___vmread (const unsigned long field,  void *ptr, const int size)
   10.65 +static always_inline int ___vmread(
   10.66 +    const unsigned long field, void *ptr, const int size)
   10.67  {
   10.68 -    unsigned long eflags;
   10.69      unsigned long ecx = 0;
   10.70 +    int rc;
   10.71  
   10.72      __asm__ __volatile__ ( VMREAD_OPCODE
   10.73 -                           MODRM_EAX_ECX       
   10.74 -                           : "=c" (ecx)
   10.75 +                           MODRM_EAX_ECX
   10.76 +                           /* CF==1 or ZF==1 --> rc = -1 */
   10.77 +                           "setna %b0 ; neg %0"
   10.78 +                           : "=r" (rc), "=c" (ecx)
   10.79                             : "a" (field)
   10.80                             : "memory");
   10.81  
   10.82 @@ -270,10 +264,7 @@ static always_inline int ___vmread (cons
   10.83          break;
   10.84      }
   10.85  
   10.86 -    __save_flags(eflags);
   10.87 -    if (eflags & X86_EFLAGS_ZF || eflags & X86_EFLAGS_CF)
   10.88 -        return -1;
   10.89 -    return 0;
   10.90 +    return rc;
   10.91  }
   10.92  
   10.93  
   10.94 @@ -315,17 +306,16 @@ static always_inline void __vmread_vcpu(
   10.95  
   10.96  static inline int __vmwrite (unsigned long field, unsigned long value)
   10.97  {
   10.98 -    unsigned long eflags;
   10.99      struct vcpu *v = current;
  10.100 +    int rc;
  10.101  
  10.102      __asm__ __volatile__ ( VMWRITE_OPCODE
  10.103                             MODRM_EAX_ECX
  10.104 -                           :
  10.105 +                           /* CF==1 or ZF==1 --> rc = -1 */
  10.106 +                           "setna %b0 ; neg %0"
  10.107 +                           : "=r" (rc)
  10.108                             : "a" (field) , "c" (value)
  10.109                             : "memory");
  10.110 -    __save_flags(eflags);
  10.111 -    if (eflags & X86_EFLAGS_ZF || eflags & X86_EFLAGS_CF)
  10.112 -        return -1;
  10.113  
  10.114      switch(field) {
  10.115      case CR0_READ_SHADOW:
  10.116 @@ -335,7 +325,7 @@ static inline int __vmwrite (unsigned lo
  10.117  	break;
  10.118      }
  10.119  
  10.120 -    return 0;
  10.121 +    return rc;
  10.122  }
  10.123  
  10.124  static inline int __vm_set_bit(unsigned long field, unsigned long mask)
  10.125 @@ -370,17 +360,17 @@ static inline void __vmxoff (void)
  10.126  
  10.127  static inline int __vmxon (u64 addr)
  10.128  {
  10.129 -    unsigned long eflags;
  10.130 +    int rc;
  10.131  
  10.132      __asm__ __volatile__ ( VMXON_OPCODE
  10.133                             MODRM_EAX_06
  10.134 -                           :
  10.135 +                           /* CF==1 or ZF==1 --> rc = -1 */
  10.136 +                           "setna %b0 ; neg %0"
  10.137 +                           : "=r" (rc)
  10.138                             : "a" (&addr) 
  10.139                             : "memory");
  10.140 -    __save_flags(eflags);
  10.141 -    if (eflags & X86_EFLAGS_ZF || eflags & X86_EFLAGS_CF)
  10.142 -        return -1;
  10.143 -    return 0;
  10.144 +
  10.145 +    return rc;
  10.146  }
  10.147  
  10.148  /* Make sure that xen intercepts any FP accesses from current */