ia64/xen-unstable

changeset 15255:bd3d6b4c52ec sparse-tree-deprecated

[XEN] Shadow: cache gva->gfn+rights translations between guest TLB flushes
to make lookups faster for emulation and hvm_copy.
Signed-off-by: Tim Deegan <Tim.Deegan@xensource.com>
author Tim Deegan <Tim.Deegan@xensource.com>
date Fri Jun 01 14:50:52 2007 +0100 (2007-06-01)
parents 13eca4bf2c69
children fee42e9bc3f0
files xen/arch/x86/mm/shadow/common.c xen/arch/x86/mm/shadow/multi.c xen/arch/x86/mm/shadow/private.h xen/arch/x86/mm/shadow/types.h xen/include/asm-x86/domain.h
line diff
     1.1 --- a/xen/arch/x86/mm/shadow/common.c	Fri Jun 01 14:32:11 2007 +0100
     1.2 +++ b/xen/arch/x86/mm/shadow/common.c	Fri Jun 01 14:50:52 2007 +0100
     1.3 @@ -2206,6 +2206,24 @@ static void sh_update_paging_modes(struc
     1.4  
     1.5      ASSERT(shadow_locked_by_me(d));
     1.6  
     1.7 +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) 
     1.8 +    /* Make sure this vcpu has a virtual TLB array allocated */
     1.9 +    if ( unlikely(!v->arch.paging.vtlb) )
    1.10 +    {
    1.11 +        v->arch.paging.vtlb = xmalloc_array(struct shadow_vtlb, VTLB_ENTRIES);
    1.12 +        if ( unlikely(!v->arch.paging.vtlb) )
    1.13 +        {
    1.14 +            SHADOW_ERROR("Could not allocate vTLB space for dom %u vcpu %u\n",
    1.15 +                         d->domain_id, v->vcpu_id);
    1.16 +            domain_crash(v->domain);
    1.17 +            return;
    1.18 +        }
    1.19 +        memset(v->arch.paging.vtlb, 0, 
    1.20 +               VTLB_ENTRIES * sizeof (struct shadow_vtlb));
    1.21 +        spin_lock_init(&v->arch.paging.vtlb_lock);
    1.22 +    }
    1.23 +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
    1.24 +
    1.25      // Valid transitions handled by this function:
    1.26      // - For PV guests:
    1.27      //     - after a shadow mode has been changed
    1.28 @@ -2514,6 +2532,18 @@ void shadow_teardown(struct domain *d)
    1.29          }
    1.30      }
    1.31  
    1.32 +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) 
    1.33 +    /* Free the virtual-TLB array attached to each vcpu */
    1.34 +    for_each_vcpu(d, v)
    1.35 +    {
    1.36 +        if ( v->arch.paging.vtlb )
    1.37 +        {
    1.38 +            xfree(v->arch.paging.vtlb);
    1.39 +            v->arch.paging.vtlb = NULL;
    1.40 +        }
    1.41 +    }
    1.42 +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
    1.43 +
    1.44      list_for_each_safe(entry, n, &d->arch.paging.shadow.p2m_freelist)
    1.45      {
    1.46          list_del(entry);
     2.1 --- a/xen/arch/x86/mm/shadow/multi.c	Fri Jun 01 14:32:11 2007 +0100
     2.2 +++ b/xen/arch/x86/mm/shadow/multi.c	Fri Jun 01 14:50:52 2007 +0100
     2.3 @@ -2997,6 +2997,11 @@ sh_invlpg(struct vcpu *v, unsigned long 
     2.4      
     2.5      perfc_incr(shadow_invlpg);
     2.6  
     2.7 +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
     2.8 +    /* No longer safe to use cached gva->gfn translations */
     2.9 +    vtlb_flush(v);
    2.10 +#endif
    2.11 +
    2.12      /* First check that we can safely read the shadow l2e.  SMP/PAE linux can
    2.13       * run as high as 6% of invlpg calls where we haven't shadowed the l2 
    2.14       * yet. */
    2.15 @@ -3057,6 +3062,7 @@ sh_invlpg(struct vcpu *v, unsigned long 
    2.16      return 1;
    2.17  }
    2.18  
    2.19 +
    2.20  static unsigned long
    2.21  sh_gva_to_gfn(struct vcpu *v, unsigned long va)
    2.22  /* Called to translate a guest virtual address to what the *guest*
    2.23 @@ -3064,11 +3070,24 @@ sh_gva_to_gfn(struct vcpu *v, unsigned l
    2.24  {
    2.25      walk_t gw;
    2.26      gfn_t gfn;
    2.27 +    
    2.28 +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
    2.29 +    struct shadow_vtlb t = {0};
    2.30 +    if ( vtlb_lookup(v, va, &t) )
    2.31 +        return t.frame_number;
    2.32 +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
    2.33  
    2.34      guest_walk_tables(v, va, &gw, 0);
    2.35      gfn = guest_walk_to_gfn(&gw);
    2.36 +
    2.37 +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
    2.38 +    t.page_number = va >> PAGE_SHIFT;
    2.39 +    t.frame_number = gfn_x(gfn);
    2.40 +    t.flags = accumulate_guest_flags(v, &gw); 
    2.41 +    vtlb_insert(v, t);
    2.42 +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
    2.43 +
    2.44      unmap_walk(v, &gw);
    2.45 -
    2.46      return gfn_x(gfn);
    2.47  }
    2.48  
    2.49 @@ -3694,6 +3713,11 @@ sh_update_cr3(struct vcpu *v, int do_loc
    2.50      /* Fix up the linear pagetable mappings */
    2.51      sh_update_linear_entries(v);
    2.52  
    2.53 +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
    2.54 +    /* No longer safe to use cached gva->gfn translations */
    2.55 +    vtlb_flush(v);
    2.56 +#endif
    2.57 +
    2.58      /* Release the lock, if we took it (otherwise it's the caller's problem) */
    2.59      if ( do_locking ) shadow_unlock(v->domain);
    2.60  }
    2.61 @@ -3918,13 +3942,41 @@ static inline void * emulate_map_dest(st
    2.62      if ( ring_3(sh_ctxt->ctxt.regs) ) 
    2.63          return NULL;
    2.64  
    2.65 -    /* Walk the guest pagetables */
    2.66 -    guest_walk_tables(v, vaddr, &gw, 1);
    2.67 -    flags = accumulate_guest_flags(v, &gw);
    2.68 -    gfn = guest_l1e_get_gfn(gw.eff_l1e);
    2.69 +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
    2.70 +    /* Try the virtual TLB first */
    2.71 +    {
    2.72 +        struct shadow_vtlb t = {0};
    2.73 +        if ( vtlb_lookup(v, vaddr, &t) 
    2.74 +             && ((t.flags & (_PAGE_PRESENT|_PAGE_RW)) 
    2.75 +                 == (_PAGE_PRESENT|_PAGE_RW)) )
    2.76 +        {
    2.77 +            flags = t.flags;
    2.78 +            gfn = _gfn(t.frame_number);
    2.79 +        }
    2.80 +        else
    2.81 +        {
    2.82 +            /* Need to do the full lookup, just in case permissions
    2.83 +             * have increased since we cached this entry */
    2.84 +            
    2.85 +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
    2.86 +
    2.87 +            /* Walk the guest pagetables */
    2.88 +            guest_walk_tables(v, vaddr, &gw, 1);
    2.89 +            flags = accumulate_guest_flags(v, &gw);
    2.90 +            gfn = guest_l1e_get_gfn(gw.eff_l1e);
    2.91 +            sh_audit_gw(v, &gw);
    2.92 +            unmap_walk(v, &gw);
    2.93 +            
    2.94 +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
    2.95 +            /* Remember this translation for next time */
    2.96 +            t.page_number = vaddr >> PAGE_SHIFT;
    2.97 +            t.frame_number = gfn_x(gfn);
    2.98 +            t.flags = flags;
    2.99 +            vtlb_insert(v, t);
   2.100 +        }
   2.101 +    }
   2.102 +#endif
   2.103      mfn = vcpu_gfn_to_mfn(v, gfn);
   2.104 -    sh_audit_gw(v, &gw);
   2.105 -    unmap_walk(v, &gw);
   2.106  
   2.107      errcode = PFEC_write_access;
   2.108      if ( !(flags & _PAGE_PRESENT) ) 
     3.1 --- a/xen/arch/x86/mm/shadow/private.h	Fri Jun 01 14:32:11 2007 +0100
     3.2 +++ b/xen/arch/x86/mm/shadow/private.h	Fri Jun 01 14:50:52 2007 +0100
     3.3 @@ -61,8 +61,9 @@ extern int shadow_audit_enable;
     3.4  #define SHOPT_PREFETCH            0x08  /* Shadow multiple entries per fault */
     3.5  #define SHOPT_LINUX_L3_TOPLEVEL   0x10  /* Pin l3es on early 64bit linux */
     3.6  #define SHOPT_SKIP_VERIFY         0x20  /* Skip PTE v'fy when safe to do so */
     3.7 +#define SHOPT_VIRTUAL_TLB         0x40  /* Cache guest v->p translations */
     3.8  
     3.9 -#define SHADOW_OPTIMIZATIONS      0x3f
    3.10 +#define SHADOW_OPTIMIZATIONS      0x7f
    3.11  
    3.12  
    3.13  /******************************************************************************
    3.14 @@ -649,6 +650,72 @@ struct x86_emulate_ops *shadow_init_emul
    3.15  void shadow_continue_emulation(
    3.16      struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
    3.17  
    3.18 +
    3.19 +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
    3.20 +/**************************************************************************/
    3.21 +/* Virtual TLB entries 
    3.22 + *
    3.23 + * We keep a cache of virtual-to-physical translations that we have seen 
    3.24 + * since the last TLB flush.  This is safe to use for frame translations, 
    3.25 + * but callers that use the rights need to re-check the actual guest tables
    3.26 + * before triggering a fault.
    3.27 + * 
    3.28 + * Lookups and updates are protected by a per-vTLB (and hence per-vcpu)
    3.29 + * lock.  This lock is held *only* while reading or writing the table,
    3.30 + * so it is safe to take in any non-interrupt context.  Most lookups
    3.31 + * happen with v==current, so we expect contention to be low.
    3.32 + */
    3.33 +
    3.34 +#define VTLB_ENTRIES 13
    3.35 +
    3.36 +struct shadow_vtlb {
    3.37 +    unsigned long page_number;    /* Guest virtual address >> PAGE_SHIFT  */
    3.38 +    unsigned long frame_number;   /* Guest physical address >> PAGE_SHIFT */
    3.39 +    u32 flags;    /* Accumulated guest pte flags, or 0 for an empty slot. */
    3.40 +};
    3.41 +
    3.42 +/* Call whenever the guest flushes hit actual TLB */
    3.43 +static inline void vtlb_flush(struct vcpu *v) 
    3.44 +{
    3.45 +    spin_lock(&v->arch.paging.vtlb_lock);
    3.46 +    memset(v->arch.paging.vtlb, 0, VTLB_ENTRIES * sizeof (struct shadow_vtlb));
    3.47 +    spin_unlock(&v->arch.paging.vtlb_lock);
    3.48 +}
    3.49 +
    3.50 +static inline int vtlb_hash(unsigned long page_number)
    3.51 +{
    3.52 +    return page_number % VTLB_ENTRIES;
    3.53 +}
    3.54 +
    3.55 +/* Put a translation into the vTLB, potentially clobbering an old one */
    3.56 +static inline void vtlb_insert(struct vcpu *v, struct shadow_vtlb entry)
    3.57 +{
    3.58 +    spin_lock(&v->arch.paging.vtlb_lock);
    3.59 +    v->arch.paging.vtlb[vtlb_hash(entry.page_number)] = entry;
    3.60 +    spin_unlock(&v->arch.paging.vtlb_lock);
    3.61 +}
    3.62 +
    3.63 +/* Look a translation up in the vTLB.  Returns 0 if not found. */
    3.64 +static inline int vtlb_lookup(struct vcpu *v, unsigned long va,
    3.65 +                              struct shadow_vtlb *result) 
    3.66 +{
    3.67 +    unsigned long page_number = va >> PAGE_SHIFT;
    3.68 +    int rv = 0;
    3.69 +    int i = vtlb_hash(page_number);
    3.70 +
    3.71 +    spin_lock(&v->arch.paging.vtlb_lock);
    3.72 +    if ( v->arch.paging.vtlb[i].flags != 0 
    3.73 +         && v->arch.paging.vtlb[i].page_number == page_number )
    3.74 +    {
    3.75 +        rv = 1; 
    3.76 +        result[0] = v->arch.paging.vtlb[i];
    3.77 +    }
    3.78 +    spin_unlock(&v->arch.paging.vtlb_lock);
    3.79 +    return rv;
    3.80 +}
    3.81 +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
    3.82 +
    3.83 +
    3.84  #endif /* _XEN_SHADOW_PRIVATE_H */
    3.85  
    3.86  /*
     4.1 --- a/xen/arch/x86/mm/shadow/types.h	Fri Jun 01 14:32:11 2007 +0100
     4.2 +++ b/xen/arch/x86/mm/shadow/types.h	Fri Jun 01 14:50:52 2007 +0100
     4.3 @@ -553,6 +553,9 @@ accumulate_guest_flags(struct vcpu *v, w
     4.4  {
     4.5      u32 accumulated_flags;
     4.6  
     4.7 +    if ( unlikely(!(guest_l1e_get_flags(gw->eff_l1e) & _PAGE_PRESENT)) )
     4.8 +        return 0;
     4.9 +        
    4.10      // We accumulate the permission flags with bitwise ANDing.
    4.11      // This works for the PRESENT bit, RW bit, and USER bit.
    4.12      // For the NX bit, however, the polarity is wrong, so we accumulate the
     5.1 --- a/xen/include/asm-x86/domain.h	Fri Jun 01 14:32:11 2007 +0100
     5.2 +++ b/xen/include/asm-x86/domain.h	Fri Jun 01 14:50:52 2007 +0100
     5.3 @@ -173,6 +173,9 @@ struct paging_vcpu {
     5.4      unsigned int translate_enabled:1;
     5.5      /* HVM guest: last emulate was to a pagetable */
     5.6      unsigned int last_write_was_pt:1;
     5.7 +    /* Translated guest: virtual TLB */    
     5.8 +    struct shadow_vtlb *vtlb;
     5.9 +    spinlock_t          vtlb_lock; 
    5.10  
    5.11      /* paging support extension */
    5.12      struct shadow_vcpu shadow;