ia64/xen-unstable

changeset 17990:9b35ae586cb8

MTRR virtualization for Intel EPT
Signed-off-by: Xiaohui Xin <xiaohui.xin@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Mon Jul 07 10:45:50 2008 +0100 (2008-07-07)
parents abd84464c09c
children 0d707feab01e
files xen/arch/x86/hvm/hvm.c xen/arch/x86/hvm/mtrr.c xen/arch/x86/hvm/vmx/vmx.c xen/arch/x86/mm/hap/p2m-ept.c xen/include/asm-x86/hvm/hvm.h xen/include/asm-x86/hvm/vmx/vmx.h xen/include/asm-x86/mtrr.h
line diff
     1.1 --- a/xen/arch/x86/hvm/hvm.c	Mon Jul 07 10:29:56 2008 +0100
     1.2 +++ b/xen/arch/x86/hvm/hvm.c	Mon Jul 07 10:45:50 2008 +0100
     1.3 @@ -836,6 +836,14 @@ static void local_flush_cache(void *info
     1.4      wbinvd();
     1.5  }
     1.6  
     1.7 +static void hvm_set_uc_mode(struct vcpu *v, bool_t is_in_uc_mode)
     1.8 +{
     1.9 +    v->domain->arch.hvm_domain.is_in_uc_mode = is_in_uc_mode;
    1.10 +    shadow_blow_tables_per_domain(v->domain);
    1.11 +    if ( hvm_funcs.set_uc_mode )
    1.12 +        return hvm_funcs.set_uc_mode(v);
    1.13 +}
    1.14 +
    1.15  int hvm_set_cr0(unsigned long value)
    1.16  {
    1.17      struct vcpu *v = current;
    1.18 @@ -923,9 +931,7 @@ int hvm_set_cr0(unsigned long value)
    1.19              {
    1.20                  /* Flush physical caches. */
    1.21                  on_each_cpu(local_flush_cache, NULL, 1, 1);
    1.22 -                /* Shadow pagetables must recognise UC mode. */
    1.23 -                v->domain->arch.hvm_domain.is_in_uc_mode = 1;
    1.24 -                shadow_blow_tables_per_domain(v->domain);
    1.25 +                hvm_set_uc_mode(v, 1);
    1.26              }
    1.27              spin_unlock(&v->domain->arch.hvm_domain.uc_lock);
    1.28          }
    1.29 @@ -937,11 +943,8 @@ int hvm_set_cr0(unsigned long value)
    1.30              v->arch.hvm_vcpu.cache_mode = NORMAL_CACHE_MODE;
    1.31  
    1.32              if ( domain_exit_uc_mode(v) )
    1.33 -            {
    1.34 -                /* Shadow pagetables must recognise normal caching mode. */
    1.35 -                v->domain->arch.hvm_domain.is_in_uc_mode = 0;
    1.36 -                shadow_blow_tables_per_domain(v->domain);
    1.37 -            }
    1.38 +                hvm_set_uc_mode(v, 0);
    1.39 +
    1.40              spin_unlock(&v->domain->arch.hvm_domain.uc_lock);
    1.41          }
    1.42      }
     2.1 --- a/xen/arch/x86/hvm/mtrr.c	Mon Jul 07 10:29:56 2008 +0100
     2.2 +++ b/xen/arch/x86/hvm/mtrr.c	Mon Jul 07 10:45:50 2008 +0100
     2.3 @@ -696,3 +696,30 @@ static int hvm_load_mtrr_msr(struct doma
     2.4  
     2.5  HVM_REGISTER_SAVE_RESTORE(MTRR, hvm_save_mtrr_msr, hvm_load_mtrr_msr,
     2.6                            1, HVMSR_PER_VCPU);
     2.7 +
     2.8 +uint8_t epte_get_entry_emt(
     2.9 +    struct domain *d, unsigned long gfn, unsigned long mfn)
    2.10 +{
    2.11 +    uint8_t gmtrr_mtype, hmtrr_mtype;
    2.12 +    uint32_t type;
    2.13 +    struct vcpu *v = current;
    2.14 +
    2.15 +    if ( (current->domain != d) && ((v = d->vcpu[0]) == NULL) )
    2.16 +        return MTRR_TYPE_WRBACK;
    2.17 +
    2.18 +    if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_IDENT_PT] )
    2.19 +        return MTRR_TYPE_WRBACK;
    2.20 +
    2.21 +    if ( (v == current) && v->domain->arch.hvm_domain.is_in_uc_mode )
    2.22 +        return MTRR_TYPE_UNCACHABLE;
    2.23 +
    2.24 +    if ( !mfn_valid(mfn) )
    2.25 +        return MTRR_TYPE_UNCACHABLE;
    2.26 +
    2.27 +    if ( hvm_get_mem_pinned_cacheattr(d, gfn, &type) )
    2.28 +        return type;
    2.29 +
    2.30 +    gmtrr_mtype = get_mtrr_type(&v->arch.hvm_vcpu.mtrr, (gfn << PAGE_SHIFT));
    2.31 +    hmtrr_mtype = get_mtrr_type(&mtrr_state, (mfn << PAGE_SHIFT));
    2.32 +    return ((gmtrr_mtype <= hmtrr_mtype) ? gmtrr_mtype : hmtrr_mtype);
    2.33 +}
     3.1 --- a/xen/arch/x86/hvm/vmx/vmx.c	Mon Jul 07 10:29:56 2008 +0100
     3.2 +++ b/xen/arch/x86/hvm/vmx/vmx.c	Mon Jul 07 10:45:50 2008 +0100
     3.3 @@ -1173,6 +1173,14 @@ static int vmx_do_pmu_interrupt(struct c
     3.4      return vpmu_do_interrupt(regs);
     3.5  }
     3.6  
     3.7 +static void vmx_set_uc_mode(struct vcpu *v)
     3.8 +{
     3.9 +    if ( paging_mode_hap(v->domain) )
    3.10 +        ept_change_entry_emt_with_range(
    3.11 +            v->domain, 0, v->domain->arch.p2m->max_mapped_pfn);
    3.12 +    vpid_sync_all();
    3.13 +}
    3.14 +
    3.15  static struct hvm_function_table vmx_function_table = {
    3.16      .name                 = "VMX",
    3.17      .domain_initialise    = vmx_domain_initialise,
    3.18 @@ -1202,7 +1210,8 @@ static struct hvm_function_table vmx_fun
    3.19      .fpu_dirty_intercept  = vmx_fpu_dirty_intercept,
    3.20      .msr_read_intercept   = vmx_msr_read_intercept,
    3.21      .msr_write_intercept  = vmx_msr_write_intercept,
    3.22 -    .invlpg_intercept     = vmx_invlpg_intercept
    3.23 +    .invlpg_intercept     = vmx_invlpg_intercept,
    3.24 +    .set_uc_mode          = vmx_set_uc_mode
    3.25  };
    3.26  
    3.27  static unsigned long *vpid_bitmap;
     4.1 --- a/xen/arch/x86/mm/hap/p2m-ept.c	Mon Jul 07 10:29:56 2008 +0100
     4.2 +++ b/xen/arch/x86/mm/hap/p2m-ept.c	Mon Jul 07 10:45:50 2008 +0100
     4.3 @@ -26,6 +26,8 @@
     4.4  #include <asm/p2m.h>
     4.5  #include <asm/hvm/vmx/vmx.h>
     4.6  #include <xen/iommu.h>
     4.7 +#include <asm/mtrr.h>
     4.8 +#include <asm/hvm/cacheattr.h>
     4.9  
    4.10  static void ept_p2m_type_to_flags(ept_entry_t *entry, p2m_type_t type)
    4.11  {
    4.12 @@ -158,8 +160,7 @@ ept_set_entry(struct domain *d, unsigned
    4.13              /* Track the highest gfn for which we have ever had a valid mapping */
    4.14              if ( gfn > d->arch.p2m->max_mapped_pfn )
    4.15                  d->arch.p2m->max_mapped_pfn = gfn;
    4.16 -
    4.17 -            ept_entry->emt = EPT_DEFAULT_MT;
    4.18 +            ept_entry->emt = epte_get_entry_emt(d, gfn, mfn_x(mfn));
    4.19              ept_entry->sp_avail = walk_level ? 1 : 0;
    4.20  
    4.21              if ( ret == GUEST_TABLE_SUPER_PAGE )
    4.22 @@ -204,11 +205,13 @@ ept_set_entry(struct domain *d, unsigned
    4.23          /* split the super page before to 4k pages */
    4.24  
    4.25          split_table = map_domain_page(ept_entry->mfn);
    4.26 +        offset = gfn & ((1 << EPT_TABLE_ORDER) - 1);
    4.27  
    4.28          for ( i = 0; i < 512; i++ )
    4.29          {
    4.30              split_ept_entry = split_table + i;
    4.31 -            split_ept_entry->emt = EPT_DEFAULT_MT;
    4.32 +            split_ept_entry->emt = epte_get_entry_emt(d,
    4.33 +                                        gfn-offset+i, split_mfn+i);
    4.34              split_ept_entry->sp_avail =  0;
    4.35  
    4.36              split_ept_entry->mfn = split_mfn+i;
    4.37 @@ -222,15 +225,13 @@ ept_set_entry(struct domain *d, unsigned
    4.38          }
    4.39  
    4.40          /* Set the destinated 4k page as normal */
    4.41 -
    4.42 -        offset = gfn & ((1 << EPT_TABLE_ORDER) - 1);
    4.43          split_ept_entry = split_table + offset;
    4.44 +        split_ept_entry->emt = epte_get_entry_emt(d, gfn, mfn_x(mfn));
    4.45          split_ept_entry->mfn = mfn_x(mfn);
    4.46          split_ept_entry->avail1 = p2mt;
    4.47          ept_p2m_type_to_flags(split_ept_entry, p2mt);
    4.48  
    4.49          unmap_domain_page(split_table);
    4.50 -
    4.51      }
    4.52  
    4.53      /* Success */
    4.54 @@ -249,7 +250,7 @@ out:
    4.55          {
    4.56              if ( order == EPT_TABLE_ORDER )
    4.57              {
    4.58 -                for ( i = 0; i < 512; i++ )
    4.59 +                for ( i = 0; i < ( 1 << order ); i++ )
    4.60                      iommu_map_page(d, gfn-offset+i, mfn_x(mfn)-offset+i);
    4.61              }
    4.62              else if ( !order )
    4.63 @@ -259,7 +260,7 @@ out:
    4.64          {
    4.65              if ( order == EPT_TABLE_ORDER )
    4.66              {
    4.67 -                for ( i = 0; i < 512; i++ )
    4.68 +                for ( i = 0; i < ( 1 << order ); i++ )
    4.69                      iommu_unmap_page(d, gfn-offset+i);
    4.70              }
    4.71              else if ( !order )
    4.72 @@ -322,11 +323,89 @@ static mfn_t ept_get_entry(struct domain
    4.73      return mfn;
    4.74  }
    4.75  
    4.76 +static uint64_t ept_get_entry_content(struct domain *d, unsigned long gfn)
    4.77 +{
    4.78 +    ept_entry_t *table =
    4.79 +        map_domain_page(mfn_x(pagetable_get_mfn(d->arch.phys_table)));
    4.80 +    unsigned long gfn_remainder = gfn;
    4.81 +    ept_entry_t *ept_entry;
    4.82 +    uint64_t content = 0;
    4.83 +
    4.84 +    u32 index;
    4.85 +    int i, ret=0;
    4.86 +
    4.87 +    /* This pfn is higher than the highest the p2m map currently holds */
    4.88 +    if ( gfn > d->arch.p2m->max_mapped_pfn )
    4.89 +        goto out;
    4.90 +
    4.91 +    for ( i = EPT_DEFAULT_GAW; i > 0; i-- )
    4.92 +    {
    4.93 +        ret = ept_next_level(d, 1, &table, &gfn_remainder,
    4.94 +                             i * EPT_TABLE_ORDER, 0);
    4.95 +        if ( !ret )
    4.96 +            goto out;
    4.97 +        else if ( ret == GUEST_TABLE_SUPER_PAGE )
    4.98 +            break;
    4.99 +    }
   4.100 +
   4.101 +    index = gfn_remainder >> ( i * EPT_TABLE_ORDER);
   4.102 +    ept_entry = table + index;
   4.103 +    content = ept_entry->epte;
   4.104 +
   4.105 + out:
   4.106 +    unmap_domain_page(table);
   4.107 +    return content;
   4.108 +}
   4.109 +
   4.110  static mfn_t ept_get_entry_current(unsigned long gfn, p2m_type_t *t)
   4.111  {
   4.112      return ept_get_entry(current->domain, gfn, t);
   4.113  }
   4.114  
   4.115 +void ept_change_entry_emt_with_range(struct domain *d, unsigned long start_gfn,
   4.116 +                 unsigned long end_gfn)
   4.117 +{
   4.118 +    unsigned long gfn;
   4.119 +    p2m_type_t p2mt;
   4.120 +    uint64_t epte;
   4.121 +    int order = 0;
   4.122 +    unsigned long mfn;
   4.123 +
   4.124 +    for ( gfn = start_gfn; gfn <= end_gfn; gfn++ )
   4.125 +    {
   4.126 +        epte = ept_get_entry_content(d, gfn);
   4.127 +        if ( epte == 0 )
   4.128 +            continue;
   4.129 +        mfn = (epte & EPTE_MFN_MASK) >> PAGE_SHIFT;
   4.130 +        if ( !mfn_valid(mfn) )
   4.131 +            continue;
   4.132 +        p2mt = (epte & EPTE_AVAIL1_MASK) >> 8;
   4.133 +        order = 0;
   4.134 +
   4.135 +        if ( epte & EPTE_SUPER_PAGE_MASK )
   4.136 +        {
   4.137 +            if ( !(gfn & ( (1 << EPT_TABLE_ORDER) - 1)) &&
   4.138 +              ((gfn + 0x1FF) <= end_gfn) )
   4.139 +            {
   4.140 +                /* gfn assigned with 2M, and the end covers more than 2m areas.
   4.141 +                 * Set emt for super page.
   4.142 +                 */
   4.143 +                order = EPT_TABLE_ORDER;
   4.144 +                ept_set_entry(d, gfn, _mfn(mfn), order, p2mt);
   4.145 +                gfn += 0x1FF;
   4.146 +            }
   4.147 +            else
   4.148 +            {
   4.149 +                /* change emt for partial entries of the 2m area */
   4.150 +                ept_set_entry(d, gfn, _mfn(mfn), order, p2mt);
   4.151 +                gfn = ((gfn >> EPT_TABLE_ORDER) << EPT_TABLE_ORDER) + 0x1FF;
   4.152 +            }
   4.153 +        }
   4.154 +        else /* gfn assigned with 4k */
   4.155 +            ept_set_entry(d, gfn, _mfn(mfn), order, p2mt);
   4.156 +    }
   4.157 +}
   4.158 +
   4.159  /* Walk the whole p2m table, changing any entries of the old type
   4.160   * to the new type.  This is used in hardware-assisted paging to
   4.161   * quickly enable or diable log-dirty tracking */
     5.1 --- a/xen/include/asm-x86/hvm/hvm.h	Mon Jul 07 10:29:56 2008 +0100
     5.2 +++ b/xen/include/asm-x86/hvm/hvm.h	Mon Jul 07 10:45:50 2008 +0100
     5.3 @@ -127,6 +127,7 @@ struct hvm_function_table {
     5.4      int (*msr_read_intercept)(struct cpu_user_regs *regs);
     5.5      int (*msr_write_intercept)(struct cpu_user_regs *regs);
     5.6      void (*invlpg_intercept)(unsigned long vaddr);
     5.7 +    void (*set_uc_mode)(struct vcpu *v);
     5.8  };
     5.9  
    5.10  extern struct hvm_function_table hvm_funcs;
     6.1 --- a/xen/include/asm-x86/hvm/vmx/vmx.h	Mon Jul 07 10:29:56 2008 +0100
     6.2 +++ b/xen/include/asm-x86/hvm/vmx/vmx.h	Mon Jul 07 10:45:50 2008 +0100
     6.3 @@ -44,6 +44,10 @@ typedef union {
     6.4  } ept_entry_t;
     6.5  
     6.6  #define EPT_TABLE_ORDER     9
     6.7 +#define EPTE_SUPER_PAGE_MASK    0x80
     6.8 +#define EPTE_MFN_MASK           0x1fffffffffff000
     6.9 +#define EPTE_AVAIL1_MASK        0xF00
    6.10 +#define EPTE_EMT_MASK           0x78
    6.11  
    6.12  void vmx_asm_vmexit_handler(struct cpu_user_regs);
    6.13  void vmx_asm_do_vmentry(void);
     7.1 --- a/xen/include/asm-x86/mtrr.h	Mon Jul 07 10:29:56 2008 +0100
     7.2 +++ b/xen/include/asm-x86/mtrr.h	Mon Jul 07 10:45:50 2008 +0100
     7.3 @@ -64,6 +64,9 @@ extern int mtrr_del_page(int reg, unsign
     7.4  extern void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi);
     7.5  extern u32 get_pat_flags(struct vcpu *v, u32 gl1e_flags, paddr_t gpaddr,
     7.6                    paddr_t spaddr);
     7.7 +extern uint8_t epte_get_entry_emt(struct domain *d, unsigned long gfn, unsigned long mfn);
     7.8 +extern void ept_change_entry_emt_with_range(struct domain *d, unsigned long start_gfn,
     7.9 +                 unsigned long end_gfn);
    7.10  extern unsigned char pat_type_2_pte_flags(unsigned char pat_type);
    7.11  
    7.12  #endif /* __ASM_X86_MTRR_H__ */