ia64/xen-unstable

changeset 17433:9153b99a7066

vmx: Enable live-migration with EPT

Signed-off-by: Xin Li <xin.b.li@intel.com>
Signed-off-by: Jun Nakajima <jun.nakajima@intel.com>
Signed-off-by: Xiaohui Xin <Xiaohui.xin@intel.com>
Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Thu Apr 10 10:00:41 2008 +0100 (2008-04-10)
parents 1d3aaa6a8b87
children 5b7a3e040683
files xen/arch/x86/hvm/vmx/vmx.c xen/arch/x86/mm/hap/hap.c xen/arch/x86/mm/hap/p2m-ept.c xen/arch/x86/mm/p2m.c xen/include/asm-x86/hvm/vmx/vmx.h xen/include/asm-x86/p2m.h
line diff
     1.1 --- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Apr 10 09:22:38 2008 +0100
     1.2 +++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Apr 10 10:00:41 2008 +0100
     1.3 @@ -2053,13 +2053,47 @@ static void vmx_wbinvd_intercept(void)
     1.4  
     1.5  static void ept_handle_violation(unsigned long qualification, paddr_t gpa)
     1.6  {
     1.7 -    if ( unlikely(((qualification >> 7) & 0x3) != 0x3) )
     1.8 +    unsigned long gla_validity = qualification & EPT_GLA_VALIDITY_MASK;
     1.9 +    struct domain *d = current->domain;
    1.10 +    unsigned long gfn = gpa >> PAGE_SHIFT;
    1.11 +    mfn_t mfn;
    1.12 +    p2m_type_t t;
    1.13 +
    1.14 +    if ( unlikely(qualification & EPT_GAW_VIOLATION) )
    1.15      {
    1.16 -        domain_crash(current->domain);
    1.17 +        gdprintk(XENLOG_ERR, "EPT violation: guest physical address %"PRIpaddr
    1.18 +                 " exceeded its width limit.\n", gpa);
    1.19 +        goto crash;
    1.20 +    }
    1.21 +
    1.22 +    if ( unlikely(gla_validity == EPT_GLA_VALIDITY_RSVD) ||
    1.23 +         unlikely(gla_validity == EPT_GLA_VALIDITY_PDPTR_LOAD) )
    1.24 +    {
    1.25 +        gdprintk(XENLOG_ERR, "EPT violation: reserved bit or "
    1.26 +                 "pdptr load violation.\n");
    1.27 +        goto crash;
    1.28 +    }
    1.29 +
    1.30 +    mfn = gfn_to_mfn(d, gfn, &t);
    1.31 +    if ( p2m_is_ram(t) && paging_mode_log_dirty(d) )
    1.32 +    {
    1.33 +        paging_mark_dirty(d, mfn_x(mfn));
    1.34 +        p2m_change_type(d, gfn, p2m_ram_logdirty, p2m_ram_rw);
    1.35 +        flush_tlb_mask(d->domain_dirty_cpumask);
    1.36          return;
    1.37      }
    1.38  
    1.39 +    /* This can only happen in log-dirty mode, writing back A/D bits. */
    1.40 +    if ( unlikely(gla_validity == EPT_GLA_VALIDITY_GPT_WALK) )
    1.41 +        goto crash;
    1.42 +
    1.43 +    ASSERT(gla_validity == EPT_GLA_VALIDITY_MATCH);
    1.44      handle_mmio();
    1.45 +
    1.46 +    return;
    1.47 +
    1.48 + crash:
    1.49 +    domain_crash(d);
    1.50  }
    1.51  
    1.52  static void vmx_failed_vmentry(unsigned int exit_reason,
     2.1 --- a/xen/arch/x86/mm/hap/hap.c	Thu Apr 10 09:22:38 2008 +0100
     2.2 +++ b/xen/arch/x86/mm/hap/hap.c	Thu Apr 10 10:00:41 2008 +0100
     2.3 @@ -62,7 +62,7 @@ int hap_enable_log_dirty(struct domain *
     2.4      hap_unlock(d);
     2.5  
     2.6      /* set l1e entries of P2M table to be read-only. */
     2.7 -    p2m_change_type_global(d, p2m_ram_rw, p2m_ram_logdirty);
     2.8 +    p2m_change_entry_type_global(d, p2m_ram_rw, p2m_ram_logdirty);
     2.9      flush_tlb_mask(d->domain_dirty_cpumask);
    2.10      return 0;
    2.11  }
    2.12 @@ -74,14 +74,14 @@ int hap_disable_log_dirty(struct domain 
    2.13      hap_unlock(d);
    2.14  
    2.15      /* set l1e entries of P2M table with normal mode */
    2.16 -    p2m_change_type_global(d, p2m_ram_logdirty, p2m_ram_rw);
    2.17 +    p2m_change_entry_type_global(d, p2m_ram_logdirty, p2m_ram_rw);
    2.18      return 0;
    2.19  }
    2.20  
    2.21  void hap_clean_dirty_bitmap(struct domain *d)
    2.22  {
    2.23      /* set l1e entries of P2M table to be read-only. */
    2.24 -    p2m_change_type_global(d, p2m_ram_rw, p2m_ram_logdirty);
    2.25 +    p2m_change_entry_type_global(d, p2m_ram_rw, p2m_ram_logdirty);
    2.26      flush_tlb_mask(d->domain_dirty_cpumask);
    2.27  }
    2.28  
     3.1 --- a/xen/arch/x86/mm/hap/p2m-ept.c	Thu Apr 10 09:22:38 2008 +0100
     3.2 +++ b/xen/arch/x86/mm/hap/p2m-ept.c	Thu Apr 10 10:00:41 2008 +0100
     3.3 @@ -26,6 +26,26 @@
     3.4  #include <asm/hvm/vmx/vmx.h>
     3.5  #include <xen/iommu.h>
     3.6  
     3.7 +static void ept_p2m_type_to_flags(ept_entry_t *entry, p2m_type_t type)
     3.8 +{
     3.9 +    switch(type)
    3.10 +    {
    3.11 +        case p2m_invalid:
    3.12 +        case p2m_mmio_dm:
    3.13 +        default:
    3.14 +            return;
    3.15 +        case p2m_ram_rw:
    3.16 +        case p2m_mmio_direct:
    3.17 +             entry->r = entry->w = entry->x = 1;
    3.18 +            return;
    3.19 +        case p2m_ram_logdirty:
    3.20 +        case p2m_ram_ro:
    3.21 +             entry->r = entry->x = 1;
    3.22 +             entry->w = 0;
    3.23 +            return;
    3.24 +    }
    3.25 +}
    3.26 +
    3.27  static int ept_next_level(struct domain *d, bool_t read_only,
    3.28                            ept_entry_t **table, unsigned long *gfn_remainder,
    3.29                            u32 shift)
    3.30 @@ -104,6 +124,7 @@ ept_set_entry(struct domain *d, unsigned
    3.31          ept_entry->avail2 = 0;
    3.32          /* last step */
    3.33          ept_entry->r = ept_entry->w = ept_entry->x = 1;
    3.34 +        ept_p2m_type_to_flags(ept_entry, p2mt);
    3.35      }
    3.36      else
    3.37          ept_entry->epte = 0;
    3.38 @@ -150,13 +171,10 @@ static mfn_t ept_get_entry(struct domain
    3.39      index = gfn_remainder;
    3.40      ept_entry = table + index;
    3.41  
    3.42 -    if ( (ept_entry->epte & 0x7) == 0x7 )
    3.43 +    if ( ept_entry->avail1 != p2m_invalid )
    3.44      {
    3.45 -        if ( ept_entry->avail1 != p2m_invalid )
    3.46 -        {
    3.47 -            *t = ept_entry->avail1;
    3.48 -            mfn = _mfn(ept_entry->mfn);
    3.49 -        }
    3.50 +        *t = ept_entry->avail1;
    3.51 +        mfn = _mfn(ept_entry->mfn);
    3.52      }
    3.53  
    3.54   out:
    3.55 @@ -169,11 +187,63 @@ static mfn_t ept_get_entry_current(unsig
    3.56      return ept_get_entry(current->domain, gfn, t);
    3.57  }
    3.58  
    3.59 +/* Walk the whole p2m table, changing any entries of the old type
    3.60 + * to the new type.  This is used in hardware-assisted paging to
    3.61 + * quickly enable or diable log-dirty tracking */
    3.62 +
    3.63 +static void ept_change_entry_type_global(struct domain *d,
    3.64 +                                            p2m_type_t ot, p2m_type_t nt)
    3.65 +{
    3.66 +    ept_entry_t *l4e, *l3e, *l2e, *l1e;
    3.67 +    int i4, i3, i2, i1;
    3.68 +
    3.69 +    if ( pagetable_get_pfn(d->arch.phys_table) == 0 )
    3.70 +        return;
    3.71 +
    3.72 +    BUG_ON(EPT_DEFAULT_GAW != 3);
    3.73 +
    3.74 +    l4e = map_domain_page(mfn_x(pagetable_get_mfn(d->arch.phys_table)));
    3.75 +    for (i4 = 0; i4 < EPT_PAGETABLE_ENTRIES; i4++ )
    3.76 +    {
    3.77 +        if ( !l4e[i4].epte || l4e[i4].sp_avail )
    3.78 +            continue;
    3.79 +        l3e = map_domain_page(l4e[i4].mfn);
    3.80 +        for ( i3 = 0; i3 < EPT_PAGETABLE_ENTRIES; i3++ )
    3.81 +        {
    3.82 +            if ( !l3e[i3].epte || l3e[i3].sp_avail )
    3.83 +                continue;
    3.84 +            l2e = map_domain_page(l3e[i3].mfn);
    3.85 +            for ( i2 = 0; i2 < EPT_PAGETABLE_ENTRIES; i2++ )
    3.86 +            {
    3.87 +                if ( !l2e[i2].epte || l2e[i2].sp_avail )
    3.88 +                    continue;
    3.89 +                l1e = map_domain_page(l2e[i2].mfn);
    3.90 +                for ( i1  = 0; i1 < EPT_PAGETABLE_ENTRIES; i1++ )
    3.91 +                {
    3.92 +                    if ( !l1e[i1].epte )
    3.93 +                        continue;
    3.94 +                    if ( l1e[i1].avail1 != ot )
    3.95 +                        continue;
    3.96 +                    l1e[i1].avail1 = nt;
    3.97 +                    ept_p2m_type_to_flags(l1e+i1, nt);
    3.98 +                }
    3.99 +                unmap_domain_page(l1e);
   3.100 +            }
   3.101 +            unmap_domain_page(l2e);
   3.102 +        }
   3.103 +        unmap_domain_page(l3e);
   3.104 +    }
   3.105 +    unmap_domain_page(l4e);
   3.106 +
   3.107 +    ept_sync_domain(d);
   3.108 +}
   3.109 +
   3.110  void ept_p2m_init(struct domain *d)
   3.111  {
   3.112      d->arch.p2m->set_entry = ept_set_entry;
   3.113      d->arch.p2m->get_entry = ept_get_entry;
   3.114      d->arch.p2m->get_entry_current = ept_get_entry_current;
   3.115 +    d->arch.p2m->change_entry_type_global = ept_change_entry_type_global;
   3.116  }
   3.117  
   3.118  /*
     4.1 --- a/xen/arch/x86/mm/p2m.c	Thu Apr 10 09:22:38 2008 +0100
     4.2 +++ b/xen/arch/x86/mm/p2m.c	Thu Apr 10 10:00:41 2008 +0100
     4.3 @@ -71,6 +71,8 @@
     4.4          spin_unlock(&(_p2m)->lock);                     \
     4.5      } while (0)
     4.6  
     4.7 +#define p2m_locked_by_me(_p2m)                            \
     4.8 +    (current->processor == (_p2m)->locker)
     4.9  
    4.10  /* Printouts */
    4.11  #define P2M_PRINTK(_f, _a...)                                \
    4.12 @@ -418,6 +420,7 @@ int p2m_init(struct domain *d)
    4.13      p2m->set_entry = p2m_set_entry;
    4.14      p2m->get_entry = p2m_gfn_to_mfn;
    4.15      p2m->get_entry_current = p2m_gfn_to_mfn_current;
    4.16 +    p2m->change_entry_type_global = p2m_change_type_global;
    4.17  
    4.18      if ( is_hvm_domain(d) && d->arch.hvm_domain.hap_enabled &&
    4.19           (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) )
    4.20 @@ -426,6 +429,16 @@ int p2m_init(struct domain *d)
    4.21      return 0;
    4.22  }
    4.23  
    4.24 +void p2m_change_entry_type_global(struct domain *d,
    4.25 +                                  p2m_type_t ot, p2m_type_t nt)
    4.26 +{
    4.27 +    struct p2m_domain *p2m = d->arch.p2m;
    4.28 +
    4.29 +    p2m_lock(p2m);
    4.30 +    p2m->change_entry_type_global(d, ot, nt);
    4.31 +    p2m_unlock(p2m);
    4.32 +}
    4.33 +
    4.34  static inline
    4.35  int set_p2m_entry(struct domain *d, unsigned long gfn, mfn_t mfn, p2m_type_t p2mt)
    4.36  {
    4.37 @@ -880,7 +893,7 @@ void p2m_change_type_global(struct domai
    4.38      if ( pagetable_get_pfn(d->arch.phys_table) == 0 )
    4.39          return;
    4.40  
    4.41 -    p2m_lock(d->arch.p2m);
    4.42 +    ASSERT(p2m_locked_by_me(d->arch.p2m));
    4.43  
    4.44  #if CONFIG_PAGING_LEVELS == 4
    4.45      l4e = map_domain_page(mfn_x(pagetable_get_mfn(d->arch.phys_table)));
    4.46 @@ -952,7 +965,6 @@ void p2m_change_type_global(struct domai
    4.47      unmap_domain_page(l2e);
    4.48  #endif
    4.49  
    4.50 -    p2m_unlock(d->arch.p2m);
    4.51  }
    4.52  
    4.53  /* Modify the p2m type of a single gfn from ot to nt, returning the 
     5.1 --- a/xen/include/asm-x86/hvm/vmx/vmx.h	Thu Apr 10 09:22:38 2008 +0100
     5.2 +++ b/xen/include/asm-x86/hvm/vmx/vmx.h	Thu Apr 10 10:00:41 2008 +0100
     5.3 @@ -344,4 +344,47 @@ void vmx_inject_nmi(struct vcpu *v);
     5.4  
     5.5  void ept_p2m_init(struct domain *d);
     5.6  
     5.7 +/* EPT violation qualifications definitions */
     5.8 +/* bit offset 0 in exit qualification */
     5.9 +#define _EPT_READ_VIOLATION         0
    5.10 +#define EPT_READ_VIOLATION          (1UL<<_EPT_READ_VIOLATION)
    5.11 +/* bit offset 1 in exit qualification */
    5.12 +#define _EPT_WRITE_VIOLATION        1
    5.13 +#define EPT_WRITE_VIOLATION         (1UL<<_EPT_WRITE_VIOLATION)
    5.14 +/* bit offset 2 in exit qualification */
    5.15 +#define _EPT_EXEC_VIOLATION         2
    5.16 +#define EPT_EXEC_VIOLATION          (1UL<<_EPT_EXEC_VIOLATION)
    5.17 +
    5.18 +/* bit offset 3 in exit qualification */
    5.19 +#define _EPT_EFFECTIVE_READ         3
    5.20 +#define EPT_EFFECTIVE_READ          (1UL<<_EPT_EFFECTIVE_READ)
    5.21 +/* bit offset 4 in exit qualification */
    5.22 +#define _EPT_EFFECTIVE_WRITE        4
    5.23 +#define EPT_EFFECTIVE_WRITE         (1UL<<_EPT_EFFECTIVE_WRITE)
    5.24 +/* bit offset 5 in exit qualification */
    5.25 +#define _EPT_EFFECTIVE_EXEC         5
    5.26 +#define EPT_EFFECTIVE_EXEC          (1UL<<_EPT_EFFECTIVE_EXEC)
    5.27 +
    5.28 +/* bit offset 6 in exit qualification */
    5.29 +#define _EPT_GAW_VIOLATION          6
    5.30 +#define EPT_GAW_VIOLATION           (1UL<<_EPT_GAW_VIOLATION)
    5.31 +
    5.32 +/* bits offset 7 & 8 in exit qualification */
    5.33 +#define _EPT_GLA_VALIDITY           7
    5.34 +#define EPT_GLA_VALIDITY_MASK       (3UL<<_EPT_GLA_VALIDITY)
    5.35 +/* gla != gpa, when load PDPTR */
    5.36 +#define EPT_GLA_VALIDITY_PDPTR_LOAD (0UL<<_EPT_GLA_VALIDITY)
    5.37 +/* gla != gpa, during guest page table walking */
    5.38 +#define EPT_GLA_VALIDITY_GPT_WALK   (1UL<<_EPT_GLA_VALIDITY)
    5.39 +/* reserved */
    5.40 +#define EPT_GLA_VALIDITY_RSVD       (2UL<<_EPT_GLA_VALIDITY)
    5.41 +/* gla == gpa, normal case */
    5.42 +#define EPT_GLA_VALIDITY_MATCH      (3UL<<_EPT_GLA_VALIDITY)
    5.43 +
    5.44 +#define EPT_EFFECTIVE_MASK          (EPT_EFFECTIVE_READ  |  \
    5.45 +                                     EPT_EFFECTIVE_WRITE |  \
    5.46 +                                     EPT_EFFECTIVE_EXEC)
    5.47 +
    5.48 +#define EPT_PAGETABLE_ENTRIES       512
    5.49 +
    5.50  #endif /* __ASM_X86_HVM_VMX_VMX_H__ */
     6.1 --- a/xen/include/asm-x86/p2m.h	Thu Apr 10 09:22:38 2008 +0100
     6.2 +++ b/xen/include/asm-x86/p2m.h	Thu Apr 10 10:00:41 2008 +0100
     6.3 @@ -107,6 +107,9 @@ struct p2m_domain {
     6.4                                         p2m_type_t *p2mt);
     6.5      mfn_t              (*get_entry_current)(unsigned long gfn,
     6.6                                              p2m_type_t *p2mt);
     6.7 +    void               (*change_entry_type_global)(struct domain *d,
     6.8 +                                                   p2m_type_t ot,
     6.9 +                                                   p2m_type_t nt);
    6.10  
    6.11      /* Highest guest frame that's ever been mapped in the p2m */
    6.12      unsigned long max_mapped_pfn;
    6.13 @@ -218,6 +221,7 @@ void guest_physmap_remove_page(struct do
    6.14  
    6.15  /* Change types across all p2m entries in a domain */
    6.16  void p2m_change_type_global(struct domain *d, p2m_type_t ot, p2m_type_t nt);
    6.17 +void p2m_change_entry_type_global(struct domain *d, p2m_type_t ot, p2m_type_t nt);
    6.18  
    6.19  /* Compare-exchange the type of a single p2m entry */
    6.20  p2m_type_t p2m_change_type(struct domain *d, unsigned long gfn,