ia64/xen-unstable

changeset 16047:e1b574bc36b5

x86/32: Re-factor mapcache data structure into per-domain and per-vcpu
components. Greatly reduces size of 'struct domain'.
Signed-off-by: Keir Fraser <keir@xensource.com>
author Keir Fraser <keir@xensource.com>
date Tue Oct 02 16:28:58 2007 +0100 (2007-10-02)
parents 818cc26e3926
children c2871913c5c2
files xen/arch/x86/domain.c xen/arch/x86/x86_32/domain_page.c xen/include/asm-x86/domain.h
line diff
     1.1 --- a/xen/arch/x86/domain.c	Tue Oct 02 13:19:25 2007 +0100
     1.2 +++ b/xen/arch/x86/domain.c	Tue Oct 02 16:28:58 2007 +0100
     1.3 @@ -382,6 +382,8 @@ int vcpu_initialise(struct vcpu *v)
     1.4  
     1.5      v->arch.flags = TF_kernel_mode;
     1.6  
     1.7 +    mapcache_vcpu_init(v);
     1.8 +
     1.9      pae_l3_cache_init(&v->arch.pae_l3_cache);
    1.10  
    1.11      paging_vcpu_init(v);
    1.12 @@ -461,7 +463,7 @@ int arch_domain_create(struct domain *d)
    1.13  
    1.14  #if defined(__i386__)
    1.15  
    1.16 -    mapcache_init(d);
    1.17 +    mapcache_domain_init(d);
    1.18  
    1.19  #else /* __x86_64__ */
    1.20  
     2.1 --- a/xen/arch/x86/x86_32/domain_page.c	Tue Oct 02 13:19:25 2007 +0100
     2.2 +++ b/xen/arch/x86/x86_32/domain_page.c	Tue Oct 02 16:28:58 2007 +0100
     2.3 @@ -43,9 +43,10 @@ static inline struct vcpu *mapcache_curr
     2.4  void *map_domain_page(unsigned long mfn)
     2.5  {
     2.6      unsigned long va;
     2.7 -    unsigned int idx, i, vcpu;
     2.8 +    unsigned int idx, i;
     2.9      struct vcpu *v;
    2.10 -    struct mapcache *cache;
    2.11 +    struct mapcache_domain *dcache;
    2.12 +    struct mapcache_vcpu *vcache;
    2.13      struct vcpu_maphash_entry *hashent;
    2.14  
    2.15      ASSERT(!in_irq());
    2.16 @@ -54,59 +55,59 @@ void *map_domain_page(unsigned long mfn)
    2.17  
    2.18      v = mapcache_current_vcpu();
    2.19  
    2.20 -    vcpu  = v->vcpu_id;
    2.21 -    cache = &v->domain->arch.mapcache;
    2.22 +    dcache = &v->domain->arch.mapcache;
    2.23 +    vcache = &v->arch.mapcache;
    2.24  
    2.25 -    hashent = &cache->vcpu_maphash[vcpu].hash[MAPHASH_HASHFN(mfn)];
    2.26 +    hashent = &vcache->hash[MAPHASH_HASHFN(mfn)];
    2.27      if ( hashent->mfn == mfn )
    2.28      {
    2.29          idx = hashent->idx;
    2.30          hashent->refcnt++;
    2.31          ASSERT(idx < MAPCACHE_ENTRIES);
    2.32          ASSERT(hashent->refcnt != 0);
    2.33 -        ASSERT(l1e_get_pfn(cache->l1tab[idx]) == mfn);
    2.34 +        ASSERT(l1e_get_pfn(dcache->l1tab[idx]) == mfn);
    2.35          goto out;
    2.36      }
    2.37  
    2.38 -    spin_lock(&cache->lock);
    2.39 +    spin_lock(&dcache->lock);
    2.40  
    2.41      /* Has some other CPU caused a wrap? We must flush if so. */
    2.42 -    if ( unlikely(cache->epoch != cache->shadow_epoch[vcpu]) )
    2.43 +    if ( unlikely(dcache->epoch != vcache->shadow_epoch) )
    2.44      {
    2.45 -        cache->shadow_epoch[vcpu] = cache->epoch;
    2.46 -        if ( NEED_FLUSH(this_cpu(tlbflush_time), cache->tlbflush_timestamp) )
    2.47 +        vcache->shadow_epoch = dcache->epoch;
    2.48 +        if ( NEED_FLUSH(this_cpu(tlbflush_time), dcache->tlbflush_timestamp) )
    2.49          {
    2.50              perfc_incr(domain_page_tlb_flush);
    2.51              local_flush_tlb();
    2.52          }
    2.53      }
    2.54  
    2.55 -    idx = find_next_zero_bit(cache->inuse, MAPCACHE_ENTRIES, cache->cursor);
    2.56 +    idx = find_next_zero_bit(dcache->inuse, MAPCACHE_ENTRIES, dcache->cursor);
    2.57      if ( unlikely(idx >= MAPCACHE_ENTRIES) )
    2.58      {
    2.59          /* /First/, clean the garbage map and update the inuse list. */
    2.60 -        for ( i = 0; i < ARRAY_SIZE(cache->garbage); i++ )
    2.61 +        for ( i = 0; i < ARRAY_SIZE(dcache->garbage); i++ )
    2.62          {
    2.63 -            unsigned long x = xchg(&cache->garbage[i], 0);
    2.64 -            cache->inuse[i] &= ~x;
    2.65 +            unsigned long x = xchg(&dcache->garbage[i], 0);
    2.66 +            dcache->inuse[i] &= ~x;
    2.67          }
    2.68  
    2.69          /* /Second/, flush TLBs. */
    2.70          perfc_incr(domain_page_tlb_flush);
    2.71          local_flush_tlb();
    2.72 -        cache->shadow_epoch[vcpu] = ++cache->epoch;
    2.73 -        cache->tlbflush_timestamp = tlbflush_current_time();
    2.74 +        vcache->shadow_epoch = ++dcache->epoch;
    2.75 +        dcache->tlbflush_timestamp = tlbflush_current_time();
    2.76  
    2.77 -        idx = find_first_zero_bit(cache->inuse, MAPCACHE_ENTRIES);
    2.78 +        idx = find_first_zero_bit(dcache->inuse, MAPCACHE_ENTRIES);
    2.79          BUG_ON(idx >= MAPCACHE_ENTRIES);
    2.80      }
    2.81  
    2.82 -    set_bit(idx, cache->inuse);
    2.83 -    cache->cursor = idx + 1;
    2.84 +    set_bit(idx, dcache->inuse);
    2.85 +    dcache->cursor = idx + 1;
    2.86  
    2.87 -    spin_unlock(&cache->lock);
    2.88 +    spin_unlock(&dcache->lock);
    2.89  
    2.90 -    l1e_write(&cache->l1tab[idx], l1e_from_pfn(mfn, __PAGE_HYPERVISOR));
    2.91 +    l1e_write(&dcache->l1tab[idx], l1e_from_pfn(mfn, __PAGE_HYPERVISOR));
    2.92  
    2.93   out:
    2.94      va = MAPCACHE_VIRT_START + (idx << PAGE_SHIFT);
    2.95 @@ -117,7 +118,7 @@ void unmap_domain_page(void *va)
    2.96  {
    2.97      unsigned int idx;
    2.98      struct vcpu *v;
    2.99 -    struct mapcache *cache;
   2.100 +    struct mapcache_domain *dcache;
   2.101      unsigned long mfn;
   2.102      struct vcpu_maphash_entry *hashent;
   2.103  
   2.104 @@ -128,11 +129,11 @@ void unmap_domain_page(void *va)
   2.105  
   2.106      v = mapcache_current_vcpu();
   2.107  
   2.108 -    cache = &v->domain->arch.mapcache;
   2.109 +    dcache = &v->domain->arch.mapcache;
   2.110  
   2.111      idx = ((unsigned long)va - MAPCACHE_VIRT_START) >> PAGE_SHIFT;
   2.112 -    mfn = l1e_get_pfn(cache->l1tab[idx]);
   2.113 -    hashent = &cache->vcpu_maphash[v->vcpu_id].hash[MAPHASH_HASHFN(mfn)];
   2.114 +    mfn = l1e_get_pfn(dcache->l1tab[idx]);
   2.115 +    hashent = &v->arch.mapcache.hash[MAPHASH_HASHFN(mfn)];
   2.116  
   2.117      if ( hashent->idx == idx )
   2.118      {
   2.119 @@ -145,10 +146,10 @@ void unmap_domain_page(void *va)
   2.120          if ( hashent->idx != MAPHASHENT_NOTINUSE )
   2.121          {
   2.122              /* /First/, zap the PTE. */
   2.123 -            ASSERT(l1e_get_pfn(cache->l1tab[hashent->idx]) == hashent->mfn);
   2.124 -            l1e_write(&cache->l1tab[hashent->idx], l1e_empty());
   2.125 +            ASSERT(l1e_get_pfn(dcache->l1tab[hashent->idx]) == hashent->mfn);
   2.126 +            l1e_write(&dcache->l1tab[hashent->idx], l1e_empty());
   2.127              /* /Second/, mark as garbage. */
   2.128 -            set_bit(hashent->idx, cache->garbage);
   2.129 +            set_bit(hashent->idx, dcache->garbage);
   2.130          }
   2.131  
   2.132          /* Add newly-freed mapping to the maphash. */
   2.133 @@ -158,30 +159,30 @@ void unmap_domain_page(void *va)
   2.134      else
   2.135      {
   2.136          /* /First/, zap the PTE. */
   2.137 -        l1e_write(&cache->l1tab[idx], l1e_empty());
   2.138 +        l1e_write(&dcache->l1tab[idx], l1e_empty());
   2.139          /* /Second/, mark as garbage. */
   2.140 -        set_bit(idx, cache->garbage);
   2.141 +        set_bit(idx, dcache->garbage);
   2.142      }
   2.143  }
   2.144  
   2.145 -void mapcache_init(struct domain *d)
   2.146 +void mapcache_domain_init(struct domain *d)
   2.147  {
   2.148 -    unsigned int i, j;
   2.149 -    struct vcpu_maphash_entry *hashent;
   2.150 -
   2.151      d->arch.mapcache.l1tab = d->arch.mm_perdomain_pt +
   2.152          (GDT_LDT_MBYTES << (20 - PAGE_SHIFT));
   2.153      spin_lock_init(&d->arch.mapcache.lock);
   2.154 +}
   2.155 +
   2.156 +void mapcache_vcpu_init(struct vcpu *v)
   2.157 +{
   2.158 +    unsigned int i;
   2.159 +    struct vcpu_maphash_entry *hashent;
   2.160  
   2.161      /* Mark all maphash entries as not in use. */
   2.162 -    for ( i = 0; i < MAX_VIRT_CPUS; i++ )
   2.163 +    for ( i = 0; i < MAPHASH_ENTRIES; i++ )
   2.164      {
   2.165 -        for ( j = 0; j < MAPHASH_ENTRIES; j++ )
   2.166 -        {
   2.167 -            hashent = &d->arch.mapcache.vcpu_maphash[i].hash[j];
   2.168 -            hashent->mfn = ~0UL; /* never valid to map */
   2.169 -            hashent->idx = MAPHASHENT_NOTINUSE;
   2.170 -        }
   2.171 +        hashent = &v->arch.mapcache.hash[i];
   2.172 +        hashent->mfn = ~0UL; /* never valid to map */
   2.173 +        hashent->idx = MAPHASHENT_NOTINUSE;
   2.174      }
   2.175  }
   2.176  
     3.1 --- a/xen/include/asm-x86/domain.h	Tue Oct 02 13:19:25 2007 +0100
     3.2 +++ b/xen/include/asm-x86/domain.h	Tue Oct 02 16:28:58 2007 +0100
     3.3 @@ -28,17 +28,21 @@ struct trap_bounce {
     3.4  #define MAPHASH_ENTRIES 8
     3.5  #define MAPHASH_HASHFN(pfn) ((pfn) & (MAPHASH_ENTRIES-1))
     3.6  #define MAPHASHENT_NOTINUSE ((u16)~0U)
     3.7 -struct vcpu_maphash {
     3.8 +struct mapcache_vcpu {
     3.9 +    /* Shadow of mapcache_domain.epoch. */
    3.10 +    unsigned int shadow_epoch;
    3.11 +
    3.12 +    /* Lock-free per-VCPU hash of recently-used mappings. */
    3.13      struct vcpu_maphash_entry {
    3.14          unsigned long mfn;
    3.15          uint16_t      idx;
    3.16          uint16_t      refcnt;
    3.17      } hash[MAPHASH_ENTRIES];
    3.18 -} __cacheline_aligned;
    3.19 +};
    3.20  
    3.21  #define MAPCACHE_ORDER   10
    3.22  #define MAPCACHE_ENTRIES (1 << MAPCACHE_ORDER)
    3.23 -struct mapcache {
    3.24 +struct mapcache_domain {
    3.25      /* The PTEs that provide the mappings, and a cursor into the array. */
    3.26      l1_pgentry_t *l1tab;
    3.27      unsigned int cursor;
    3.28 @@ -47,27 +51,25 @@ struct mapcache {
    3.29      spinlock_t lock;
    3.30  
    3.31      /* Garbage mappings are flushed from TLBs in batches called 'epochs'. */
    3.32 -    unsigned int epoch, shadow_epoch[MAX_VIRT_CPUS];
    3.33 +    unsigned int epoch;
    3.34      u32 tlbflush_timestamp;
    3.35  
    3.36      /* Which mappings are in use, and which are garbage to reap next epoch? */
    3.37      unsigned long inuse[BITS_TO_LONGS(MAPCACHE_ENTRIES)];
    3.38      unsigned long garbage[BITS_TO_LONGS(MAPCACHE_ENTRIES)];
    3.39 -
    3.40 -    /* Lock-free per-VCPU hash of recently-used mappings. */
    3.41 -    struct vcpu_maphash vcpu_maphash[MAX_VIRT_CPUS];
    3.42  };
    3.43  
    3.44 -extern void mapcache_init(struct domain *);
    3.45 +void mapcache_domain_init(struct domain *);
    3.46 +void mapcache_vcpu_init(struct vcpu *);
    3.47  
    3.48  /* x86/64: toggle guest between kernel and user modes. */
    3.49 -extern void toggle_guest_mode(struct vcpu *);
    3.50 +void toggle_guest_mode(struct vcpu *);
    3.51  
    3.52  /*
    3.53   * Initialise a hypercall-transfer page. The given pointer must be mapped
    3.54   * in Xen virtual address space (accesses are not validated or checked).
    3.55   */
    3.56 -extern void hypercall_page_initialise(struct domain *d, void *);
    3.57 +void hypercall_page_initialise(struct domain *d, void *);
    3.58  
    3.59  /************************************************/
    3.60  /*          shadow paging extension             */
    3.61 @@ -204,7 +206,7 @@ struct arch_domain
    3.62  
    3.63  #ifdef CONFIG_X86_32
    3.64      /* map_domain_page() mapping cache. */
    3.65 -    struct mapcache mapcache;
    3.66 +    struct mapcache_domain mapcache;
    3.67  #endif
    3.68  
    3.69  #ifdef CONFIG_COMPAT
    3.70 @@ -290,7 +292,7 @@ struct arch_vcpu
    3.71      struct trap_bounce trap_bounce;
    3.72  
    3.73      /* I/O-port access bitmap. */
    3.74 -    XEN_GUEST_HANDLE(uint8_t) iobmp; /* Guest kernel virtual address of the bitmap. */
    3.75 +    XEN_GUEST_HANDLE(uint8_t) iobmp; /* Guest kernel vaddr of the bitmap. */
    3.76      int iobmp_limit;  /* Number of ports represented in the bitmap.  */
    3.77      int iopl;         /* Current IOPL for this VCPU. */
    3.78  
    3.79 @@ -327,6 +329,12 @@ struct arch_vcpu
    3.80  
    3.81      /* Guest-specified relocation of vcpu_info. */
    3.82      unsigned long vcpu_info_mfn;
    3.83 +
    3.84 +#ifdef CONFIG_X86_32
    3.85 +    /* map_domain_page() mapping cache. */
    3.86 +    struct mapcache_vcpu mapcache;
    3.87 +#endif
    3.88 +
    3.89  } __cacheline_aligned;
    3.90  
    3.91  /* Shorthands to improve code legibility. */